Linux: on every PR, run make -C deps USE_BINARYBUILDER=0 and make sure that the working directory is clean

pipelines/main/launch_unsigned_jobs.yml

@@ -60,10 +60,18 @@ steps:
           buildkite-agent pipeline upload .buildkite/pipelines/main/misc/pdf_docs/build_pdf_docs.yml
           buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml
           buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml
-          buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml
 
+          # Sanitizers
           buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/asan.yml
           buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/tsan.yml
+
+          bash .buildkite/utilities/arches_pipeline_upload.sh \
+              .buildkite/pipelines/main/misc/srcdepchecksums/linux.arches \
+              .buildkite/pipelines/main/misc/srcdepchecksums/linux.yml
+
+          # With Buildkite, the last uploaded pipeline (within a step) actually gets started first.
+          # So we launch `whitespace` last so that it gets started first.
+          buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml
         agents:
           queue: "julia"
           os: "linux"

pipelines/main/misc/srcdepchecksums/linux.arches

@@ -0,0 +1,13 @@
+# ROOTFS_IMAGE_NAME    TRIPLET                    ARCH           ARCH_ROOTFS    TIMEOUT       ROOTFS_TAG    ROOTFS_HASH
+llvm_passes            i686-linux-gnu             x86_64         i686           .             v5.51         0000000000000000000000000000000000000000
+llvm_passes            x86_64-linux-gnu           x86_64         x86_64         .             v5.51         0000000000000000000000000000000000000000
+llvm_passes            x86_64-linux-gnuassert     x86_64         x86_64         .             v5.51         0000000000000000000000000000000000000000
+llvm_passes            aarch64-linux-gnu          aarch64        aarch64        .             v5.51         0000000000000000000000000000000000000000
+llvm_passes            powerpc64le-linux-gnu      powerpc64le    powerpc64le    .             v5.51         0000000000000000000000000000000000000000
+package_musl           x86_64-linux-musl          x86_64         x86_64         .             v5.51         0000000000000000000000000000000000000000
+# llvm_passes          armv7l-linux-gnueabihf     armv7l         armv7l         .             ----          0000000000000000000000000000000000000000
+
+# These special lines allow us to embed default values for the columns above.
+# Any column without a default mapping here will simply substitute a `.` to the empty string
+
+#default TIMEOUT 30

pipelines/main/misc/srcdepchecksums/linux.yml

@@ -0,0 +1,32 @@
+steps:
+  - group: "Check"
+    steps:
+      - label: ":linux: srcdepchecksums ${TRIPLET?}"
+        key: "srcdepchecksums_${TRIPLET?}"
+        plugins:
+          - JuliaCI/external-buildkite#v1:
+              version: "./.buildkite-external-version"
+              repo_url: "https://github.com/JuliaCI/julia-buildkite"
+          - JuliaCI/julia#v1:
+              # Drop default "registries" directory, so it is not persisted from execution to execution
+              persist_depot_dirs: packages,artifacts,compiled
+              version: '1.6'
+          - staticfloat/sandbox#v1:
+              rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/${ROOTFS_IMAGE_NAME?}.${ARCH_ROOTFS?}.tar.gz
+              rootfs_treehash: "${ROOTFS_HASH?}"
+              uid: 1000
+              gid: 1000
+              workspaces:
+                # Include `/cache/repos` so that our `git` version introspection works.
+                - "/cache/repos:/cache/repos"
+        timeout_in_minutes: ${TIMEOUT?}
+        commands: "bash .buildkite/utilities/make_deps_no_bb.sh"
+        agents:
+          queue: "julia"
+          # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
+          sandbox_capable: "true"
+          os: "linux"
+          arch: "${ARCH?}"
+        env:
+          # Receive some environment mappings from our templating engine
+          TRIPLET: "${TRIPLET?}"

pipelines/main/platforms/build_linux.soft_fail.arches

@@ -1,5 +1,5 @@
 # ROOTFS_IMAGE_NAME    TRIPLET                   ARCH           ARCH_ROOTFS    MAKE_FLAGS    TIMEOUT     ROOTFS_TAG    ROOTFS_HASH
-# package_linux        armv7l-linux-gnueabihf    armv7l         armv7l         .             .           ----          ----------------------------------------
+# package_linux        armv7l-linux-gnueabihf    armv7l         armv7l         .             .           ----          0000000000000000000000000000000000000000
 
 # These special lines allow us to embed default values for the columns above.
 # Any column without a default mapping here will simply substitute a `.` to the empty string

utilities/build_julia.sh

@@ -47,14 +47,7 @@ done
 echo "--- Build Julia"
 make "${MFLAGS[@]}"
 
-
-echo "--- Check that the working directory is clean"
-if [ -n "$(git status --short)" ]; then
-    echo "ERROR: The working directory is dirty." >&2
-    echo "Output of git status:" >&2
-    git status
-    exit 1
-fi
+source .buildkite/utilities/check_clean_wd.sh
 
 echo "--- Print Julia version info"
 # use `JULIA_BINARY` since it has the `.exe` extension already determined,

utilities/check_clean_wd.sh

@@ -0,0 +1,7 @@
+echo "--- Check that the working directory is clean"
+if [ -n "$(git status --short)" ]; then
+    echo "ERROR: The working directory is dirty." >&2
+    echo "Output of git status:" >&2
+    git status
+    exit 1
+fi

utilities/make_deps_no_bb.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# This script does the following:
+# 1. make -C deps USE_BINARYBUILDER=0
+# 2. Make sure that the working directory is clean.
+#
+# The purpose of this script is to make sure that the from-source (non-BinaryBuilder)
+# checksums are all up to date.
+
+set -euo pipefail
+
+make -C deps USE_BINARYBUILDER=0
+
+source .buildkite/utilities/check_clean_wd.sh