From c78a436dac6beff27cfc7c62c157ab6bdfc7287f Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Wed, 30 Nov 2022 20:01:48 -0500 Subject: [PATCH 1/2] Linux: on every PR, run `make -C deps USE_BINARYBUILDER=0` and make sure that the working directory is clean --- pipelines/main/launch_unsigned_jobs.yml | 10 +++++- .../main/misc/srcdepchecksums/linux.arches | 13 ++++++++ pipelines/main/misc/srcdepchecksums/linux.yml | 32 +++++++++++++++++++ .../platforms/build_linux.soft_fail.arches | 2 +- utilities/build_julia.sh | 9 +----- utilities/check_clean_wd.sh | 7 ++++ utilities/make_deps_no_bb.sh | 14 ++++++++ 7 files changed, 77 insertions(+), 10 deletions(-) create mode 100644 pipelines/main/misc/srcdepchecksums/linux.arches create mode 100644 pipelines/main/misc/srcdepchecksums/linux.yml create mode 100644 utilities/check_clean_wd.sh create mode 100644 utilities/make_deps_no_bb.sh diff --git a/pipelines/main/launch_unsigned_jobs.yml b/pipelines/main/launch_unsigned_jobs.yml index eb3923ea..f9e8e643 100644 --- a/pipelines/main/launch_unsigned_jobs.yml +++ b/pipelines/main/launch_unsigned_jobs.yml @@ -60,10 +60,18 @@ steps: buildkite-agent pipeline upload .buildkite/pipelines/main/misc/pdf_docs/build_pdf_docs.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml + # Sanitizers buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/asan.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/tsan.yml + + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/misc/srcdepchecksums/linux.arches \ + .buildkite/pipelines/main/misc/srcdepchecksums/linux.yml + + # With Buildkite, the last uploaded pipeline (within a step) actually gets started first. + # So we launch `whitespace` last so that it gets started first. + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml agents: queue: "julia" os: "linux" diff --git a/pipelines/main/misc/srcdepchecksums/linux.arches b/pipelines/main/misc/srcdepchecksums/linux.arches new file mode 100644 index 00000000..9c5a15be --- /dev/null +++ b/pipelines/main/misc/srcdepchecksums/linux.arches @@ -0,0 +1,13 @@ +# ROOTFS_IMAGE_NAME TRIPLET ARCH ARCH_ROOTFS TIMEOUT ROOTFS_TAG ROOTFS_HASH +llvm_passes i686-linux-gnu x86_64 i686 . v5.50 0000000000000000000000000000000000000000 +llvm_passes x86_64-linux-gnu x86_64 x86_64 . v5.50 0000000000000000000000000000000000000000 +llvm_passes x86_64-linux-gnuassert x86_64 x86_64 . v5.50 0000000000000000000000000000000000000000 +llvm_passes aarch64-linux-gnu aarch64 aarch64 . v5.50 0000000000000000000000000000000000000000 +llvm_passes powerpc64le-linux-gnu powerpc64le powerpc64le . v5.50 0000000000000000000000000000000000000000 +package_musl x86_64-linux-musl x86_64 x86_64 . v5.50 0000000000000000000000000000000000000000 +# llvm_passes armv7l-linux-gnueabihf armv7l armv7l . ---- 0000000000000000000000000000000000000000 + +# These special lines allow us to embed default values for the columns above. +# Any column without a default mapping here will simply substitute a `.` to the empty string + +#default TIMEOUT 30 diff --git a/pipelines/main/misc/srcdepchecksums/linux.yml b/pipelines/main/misc/srcdepchecksums/linux.yml new file mode 100644 index 00000000..60b498c7 --- /dev/null +++ b/pipelines/main/misc/srcdepchecksums/linux.yml @@ -0,0 +1,32 @@ +steps: + - group: "Check" + steps: + - label: ":linux: srcdepchecksums ${TRIPLET?}" + key: "srcdepchecksums_${TRIPLET?}" + plugins: + - JuliaCI/external-buildkite#v1: + version: "./.buildkite-external-version" + repo_url: "https://github.com/JuliaCI/julia-buildkite" + - JuliaCI/julia#v1: + # Drop default "registries" directory, so it is not persisted from execution to execution + persist_depot_dirs: packages,artifacts,compiled + version: '1.6' + - staticfloat/sandbox#v1: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/${ROOTFS_IMAGE_NAME?}.${ARCH_ROOTFS?}.tar.gz + rootfs_treehash: "${ROOTFS_HASH?}" + uid: 1000 + gid: 1000 + workspaces: + # Include `/cache/repos` so that our `git` version introspection works. + - "/cache/repos:/cache/repos" + timeout_in_minutes: ${TIMEOUT?} + commands: "bash .buildkite/utilities/make_deps_no_bb.sh" + agents: + queue: "julia" + # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing + sandbox_capable: "true" + os: "linux" + arch: "${ARCH?}" + env: + # Receive some environment mappings from our templating engine + TRIPLET: "${TRIPLET?}" diff --git a/pipelines/main/platforms/build_linux.soft_fail.arches b/pipelines/main/platforms/build_linux.soft_fail.arches index 45aeaaf7..0886ac8d 100644 --- a/pipelines/main/platforms/build_linux.soft_fail.arches +++ b/pipelines/main/platforms/build_linux.soft_fail.arches @@ -1,5 +1,5 @@ # ROOTFS_IMAGE_NAME TRIPLET ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT ROOTFS_TAG ROOTFS_HASH -# package_linux armv7l-linux-gnueabihf armv7l armv7l . . ---- ---------------------------------------- +# package_linux armv7l-linux-gnueabihf armv7l armv7l . . ---- 0000000000000000000000000000000000000000 # These special lines allow us to embed default values for the columns above. # Any column without a default mapping here will simply substitute a `.` to the empty string diff --git a/utilities/build_julia.sh b/utilities/build_julia.sh index c0b5dac5..17d3c38a 100755 --- a/utilities/build_julia.sh +++ b/utilities/build_julia.sh @@ -47,14 +47,7 @@ done echo "--- Build Julia" make "${MFLAGS[@]}" - -echo "--- Check that the working directory is clean" -if [ -n "$(git status --short)" ]; then - echo "ERROR: The working directory is dirty." >&2 - echo "Output of git status:" >&2 - git status - exit 1 -fi +source .buildkite/utilities/check_clean_wd.sh echo "--- Print Julia version info" # use `JULIA_BINARY` since it has the `.exe` extension already determined, diff --git a/utilities/check_clean_wd.sh b/utilities/check_clean_wd.sh new file mode 100644 index 00000000..6bf03f0e --- /dev/null +++ b/utilities/check_clean_wd.sh @@ -0,0 +1,7 @@ +echo "--- Check that the working directory is clean" +if [ -n "$(git status --short)" ]; then + echo "ERROR: The working directory is dirty." >&2 + echo "Output of git status:" >&2 + git status + exit 1 +fi diff --git a/utilities/make_deps_no_bb.sh b/utilities/make_deps_no_bb.sh new file mode 100644 index 00000000..2434c605 --- /dev/null +++ b/utilities/make_deps_no_bb.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# This script does the following: +# 1. make -C deps USE_BINARYBUILDER=0 +# 2. Make sure that the working directory is clean. +# +# The purpose of this script is to make sure that the from-source (non-BinaryBuilder) +# checksums are all up to date. + +set -euo pipefail + +make -C deps USE_BINARYBUILDER=0 + +source .buildkite/utilities/check_clean_wd.sh From fd26c74488d0f74f646070a259853965970fde8a Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Thu, 1 Dec 2022 01:56:29 -0500 Subject: [PATCH 2/2] Update linux.arches --- pipelines/main/misc/srcdepchecksums/linux.arches | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pipelines/main/misc/srcdepchecksums/linux.arches b/pipelines/main/misc/srcdepchecksums/linux.arches index 9c5a15be..e3020ccb 100644 --- a/pipelines/main/misc/srcdepchecksums/linux.arches +++ b/pipelines/main/misc/srcdepchecksums/linux.arches @@ -1,10 +1,10 @@ # ROOTFS_IMAGE_NAME TRIPLET ARCH ARCH_ROOTFS TIMEOUT ROOTFS_TAG ROOTFS_HASH -llvm_passes i686-linux-gnu x86_64 i686 . v5.50 0000000000000000000000000000000000000000 -llvm_passes x86_64-linux-gnu x86_64 x86_64 . v5.50 0000000000000000000000000000000000000000 -llvm_passes x86_64-linux-gnuassert x86_64 x86_64 . v5.50 0000000000000000000000000000000000000000 -llvm_passes aarch64-linux-gnu aarch64 aarch64 . v5.50 0000000000000000000000000000000000000000 -llvm_passes powerpc64le-linux-gnu powerpc64le powerpc64le . v5.50 0000000000000000000000000000000000000000 -package_musl x86_64-linux-musl x86_64 x86_64 . v5.50 0000000000000000000000000000000000000000 +llvm_passes i686-linux-gnu x86_64 i686 . v5.51 0000000000000000000000000000000000000000 +llvm_passes x86_64-linux-gnu x86_64 x86_64 . v5.51 0000000000000000000000000000000000000000 +llvm_passes x86_64-linux-gnuassert x86_64 x86_64 . v5.51 0000000000000000000000000000000000000000 +llvm_passes aarch64-linux-gnu aarch64 aarch64 . v5.51 0000000000000000000000000000000000000000 +llvm_passes powerpc64le-linux-gnu powerpc64le powerpc64le . v5.51 0000000000000000000000000000000000000000 +package_musl x86_64-linux-musl x86_64 x86_64 . v5.51 0000000000000000000000000000000000000000 # llvm_passes armv7l-linux-gnueabihf armv7l armv7l . ---- 0000000000000000000000000000000000000000 # These special lines allow us to embed default values for the columns above.