Update concurrency rules and limit build queue to one

.buildkite/build_pipeline.yml

@@ -1,33 +1,45 @@
 steps:
-  - input: "Build parameters"
-    if: build.source == "ui"
-    fields:
-      - select: "Rebuild?"
-        key: "REBUILD"
-        default: ""
-        required: false
-        options:
-          - label: "no"
-            value: ""
-          - label: "yes"
-            value: "rebuild"
-        hint: "Should all books be rebuilt, regardless of what has changed? Build once with this set to true after every release."
-      - select: "How should broken links be handled?"
-        key: "BROKEN_LINKS"
-        default: ""
-        required: false
-        options:
-          - label: "Continue without warning"
-            value: "skiplinkcheck"
-          - label: "Continue, but log a warning"
-            value: "warnlinkcheck"
-          - label: "Fail the build"
-            value: ""
-        hint: "Should we ignore checking broken links? Should we allow to run the build without failing if there's a broken link? Ignoring broken links is dangerous not just because bad links will leak into the public site but because subsequent builds and pull requests that do not fix the links fail."
+  # - input: "Build parameters"
+  #   if: build.source == "ui"
+  #   fields:
+  #     - select: "Rebuild?"
+  #       key: "REBUILD"
+  #       default: ""
+  #       required: false
+  #       options:
+  #         - label: "no"
+  #           value: ""
+  #         - label: "yes"
+  #           value: "rebuild"
+  #       hint: "Should all books be rebuilt, regardless of what has changed? Build once with this set to true after every release."
+  #     - select: "How should broken links be handled?"
+  #       key: "BROKEN_LINKS"
+  #       default: ""
+  #       required: false
+  #       options:
+  #         - label: "Continue without warning"
+  #           value: "skiplinkcheck"
+  #         - label: "Continue, but log a warning"
+  #           value: "warnlinkcheck"
+  #         - label: "Fail the build"
+  #           value: ""
+  #       hint: "Should we ignore checking broken links? Should we allow to run the build without failing if there's a broken link? Ignoring broken links is dangerous not just because bad links will leak into the public site but because subsequent builds and pull requests that do not fix the links fail."
+  # - wait
+    # This step is purposefully outside of the concurrency gate
+  - label: "Check for queue"
+    key: check-queue
+    # if: build.source == "schedule"
+    command: ".buildkite/scripts/check_queue.sh"
+  - wait
+  - label: "Start of concurrency gate"
+    command: echo "Start of concurrency gate"
+    concurrency_group: gate/docs-build-${BUILDKITE_BRANCH}
+    concurrency: 1
   - wait
   - label: "Full rebuild or incremental build?"
-    if: build.source == "schedule"
+    # if: build.source == "schedule"
     command: ".buildkite/scripts/compare_commits.sh"
+  - wait
   - label: ":white_check_mark: Build docs"
     command: |
       export REBUILD="$(buildkite-agent meta-data get REBUILD --default '' --log-level fatal)"
@@ -37,7 +49,10 @@ steps:
       provider: "gcp"
       image: family/docs-ubuntu-2204
       machineType: ${BUILD_MACHINE_TYPE}
-    concurrency_group: build-docs-${BUILDKITE_BRANCH}
+  - wait
+  - label: "End of concurrency gate"
+    command: echo "End of concurrency gate"
+    concurrency_group: gate/docs-build-${BUILDKITE_BRANCH}
     concurrency: 1
 notify:
   - email: "[email protected]"

.buildkite/scripts/check_queue.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+build_data_url="https://api.buildkite.com/v2/organizations/elastic/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds?branch=${BUILDKITE_BRANCH}"
+cancel_build_url="https://api.buildkite.com/v2/organizations/elastic/pipelines/${BUILDKITE_PIPELINE_SLUG}/builds/${BUILDKITE_BUILD_NUMBER}/cancel"
+
+# Don't look at this build (it's running now!)
+# Don't look at the last build (it's okay if it's still running!)
+# Look three builds back instead (if this build is still running,
+# it means there's already one in the queue and we can safely cancel this one)
+THIRD_TO_LAST_BUILD_STATE=$(curl -s -H "Authorization: Bearer ${BUILDKITE_API_TOKEN}" $build_data_url | jq -r '.[2].state')
+
+echo "Determining if there are multiple builds waiting."
+if [[ "$THIRD_TO_LAST_BUILD_STATE" == "running" ]]; then
+  echo "The pipeline is congested. Canceling this build."
+  curl -sX PUT -H "Authorization: Bearer ${BUILDKITE_API_TOKEN}" $cancel_build_url
+else
+  echo "The pipeline is ready for a new build."
+fi