From bc4f91b7cbfabc11491853ed0f805af541519ea3 Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Wed, 2 Oct 2024 12:12:03 +0200 Subject: [PATCH] Retry apt-get and rustup on CI `Setup rust toolchain` build step was observed to be flaky. Retries may help. --- .github/actions/setup-builder/action.yaml | 12 +++++++----- ci/scripts/retry | 21 +++++++++++++++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) create mode 100755 ci/scripts/retry diff --git a/.github/actions/setup-builder/action.yaml b/.github/actions/setup-builder/action.yaml index 5578517ec359..6e017a920a8c 100644 --- a/.github/actions/setup-builder/action.yaml +++ b/.github/actions/setup-builder/action.yaml @@ -28,16 +28,18 @@ runs: - name: Install Build Dependencies shell: bash run: | - apt-get update - apt-get install -y protobuf-compiler + RETRY=ci/bin/retry + "${RETRY}" apt-get update + "${RETRY}" apt-get install -y protobuf-compiler - name: Setup Rust toolchain shell: bash # rustfmt is needed for the substrait build script run: | + RETRY=ci/bin/retry echo "Installing ${{ inputs.rust-version }}" - rustup toolchain install ${{ inputs.rust-version }} - rustup default ${{ inputs.rust-version }} - rustup component add rustfmt + "${RETRY}" rustup toolchain install ${{ inputs.rust-version }} + "${RETRY}" rustup default ${{ inputs.rust-version }} + "${RETRY}" rustup component add rustfmt - name: Configure rust runtime env uses: ./.github/actions/setup-rust-runtime - name: Fixup git permissions diff --git a/ci/scripts/retry b/ci/scripts/retry new file mode 100755 index 000000000000..9e9bd9da4d62 --- /dev/null +++ b/ci/scripts/retry @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -euo pipefail + +x() { + echo "+ $*" >&2 + "$@" +} + +max_retry_time_seconds=$(( 30 * 60 )) +retry_delay_seconds=10 + +END=$(( $(date +%s) + ${max_retry_time_seconds} )) + +while (( $(date +%s) < $END )); do + x "$@" && exit 0 + sleep "${retry_delay_seconds}" +done + +echo "$0: retrying [$*] timed out" >&2 +exit 1