From df34faa4b64a68c918b9d63a9d3408636fd4e93c Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 12 Oct 2024 00:03:40 +0000 Subject: [PATCH 1/5] try to fix runner label --- .github/workflows/ci.yml | 215 ++++++++++++++++++++++----------------- 1 file changed, 120 insertions(+), 95 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d7167fdcf2..96086f3e9c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,10 @@ name: CI on: push: - branches: [master, provernet] + branches: + - master + - provernet + - '*/ci-push*' pull_request: types: - opened @@ -22,7 +25,6 @@ env: DOCKERHUB_PASSWORD: "${{ secrets.DOCKERHUB_PASSWORD }}" RUN_ID: ${{ github.run_id }} RUN_ATTEMPT: ${{ github.run_attempt }} - USERNAME: ${{ github.event.pull_request.user.login || github.actor }} GITHUB_TOKEN: ${{ github.token }} GH_SELF_HOSTED_RUNNER_TOKEN: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -41,14 +43,16 @@ jobs: runner_type: builder-x86 secrets: inherit - changes: + configure: runs-on: ubuntu-20.04 if: github.event.pull_request.draft == false # Required permissions. permissions: pull-requests: read - # Set job outputs to values from filter step + # Set job outputs to values from filter step. + # Set username to a value derived from our github username. outputs: + username: ${{ steps.compute_username.outputs.username }} avm-transpiler: ${{ steps.filter.outputs.avm-transpiler }} build-images: ${{ steps.filter.outputs.build-images }} barretenberg: ${{ steps.filter.outputs.barretenberg }} @@ -61,6 +65,19 @@ jobs: non-misc-ci: ${{ steps.filter.outputs.non-misc-ci }} non-barretenberg-cpp: ${{ steps.filter.outputs.non-barretenberg-cpp }} steps: + - name: Compute Username + id: compute_username + shell: bash + run: | + if [ "${{ github.event_name }}" == "pull_request" ]; then + echo "username=${{ github.event.pull_request.user.login }}" >> $GITHUB_OUTPUT + else + GIT_HASH="${{ github.sha }}" + GIT_HASH_LAST8=${GIT_HASH: -8} + GIT_HASH_LAST8_DEC=$(printf "%d" 0x$GIT_HASH_LAST8) + GIT_HASH_MODULO_8=$((GIT_HASH_LAST8_DEC % 8)) + echo "username=master-${GIT_HASH_MODULO_8}" >> $GITHUB_OUTPUT + fi - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 @@ -99,28 +116,27 @@ jobs: - .github/workflows/ci.yml build-images: - needs: [setup, changes] + needs: [setup, configure] # Note we don't but the 'if' here as that would also apply to dependent jobs, instead we just copy it into each step - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + runs-on: ${{ needs.configure.outputs.username }}-x86 steps: - uses: actions/checkout@v4 - if: needs.changes.outputs.build-images == 'true' + if: needs.configure.outputs.build-images == 'true' with: { ref: "${{ env.GIT_COMMIT }}" } - uses: ./.github/ci-setup-action - if: needs.changes.outputs.build-images == 'true' + if: needs.configure.outputs.build-images == 'true' with: concurrency_key: build-images-x86 - name: "Push Build Images If Changed" - if: needs.changes.outputs.build-images == 'true' + if: needs.configure.outputs.build-images == 'true' timeout-minutes: 40 run: | earthly-ci --push ./build-images/+build build: - needs: [build-images, changes] - # NOTE: we don't skip this job here instead using if's below so that dependencies are not broken - # we still want build to be at the bottom of the dependency chain - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + needs: [build-images, configure] + # NOTE: we don't skip this job here instead using if's below to avoid gotcha if dependent jobs have their own if's + runs-on: ${{ needs.configure.outputs.username }}-x86 outputs: e2e_list: ${{ steps.e2e_list.outputs.list }} bench_list: ${{ steps.bench_list.outputs.list }} @@ -133,7 +149,7 @@ jobs: # prepare images locally, tagged by commit hash - name: "Build E2E Image" timeout-minutes: 40 - if: (needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true' && needs.changes.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master' + if: (needs.configure.outputs.non-docs == 'true' && needs.configure.outputs.non-misc-ci == 'true' && needs.configure.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master' run: | earthly-ci ./yarn-project+export-e2e-test-images # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end @@ -161,8 +177,8 @@ jobs: # all the non-bench end-to-end integration tests for aztec e2e: - needs: [build, changes] - if: (needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true' && needs.changes.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'e2e') + needs: [build, configure] + if: (needs.configure.outputs.non-docs == 'true' && needs.configure.outputs.non-misc-ci == 'true' && needs.configure.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'e2e') runs-on: ubuntu-20.04 strategy: fail-fast: false @@ -175,6 +191,8 @@ jobs: - name: Setup and Test timeout-minutes: 40 uses: ./.github/ensure-tester-with-images + env: + USERNAME: ${{ needs.configure.outputs.username }} with: runner_type: ${{ contains(matrix.test, 'prover') && '64core-tester-x86' || '8core-tester-x86' }} builder_type: builder-x86 @@ -191,8 +209,8 @@ jobs: # all the benchmarking end-to-end integration tests for aztec (not required to merge) bench-e2e: - needs: [build, changes] - if: (needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true' && needs.changes.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'bench') + needs: [build, configure] + if: (needs.configure.outputs.non-docs == 'true' && needs.configure.outputs.non-misc-ci == 'true' && needs.configure.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'bench') runs-on: ubuntu-20.04 strategy: fail-fast: false @@ -204,6 +222,8 @@ jobs: - uses: ./.github/ci-setup-action - name: Setup and Test uses: ./.github/ensure-tester-with-images + env: + USERNAME: ${{ needs.configure.outputs.username }} timeout-minutes: 45 with: runner_type: ${{ contains(matrix.test, 'prover') && '64core-tester-x86' || '16core-tester-x86' }} @@ -223,15 +243,17 @@ jobs: acir-bench: runs-on: ubuntu-20.04 - needs: [build, changes] + needs: [build, configure] # Note: not fully accurate, but to work with bench-summary needs to be the same as bench-e2e - if: needs.changes.outputs.non-barretenberg-cpp == 'true' + if: needs.configure.outputs.non-barretenberg-cpp == 'true' steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } - uses: ./.github/ci-setup-action - name: Setup and Test uses: ./.github/ensure-tester-with-images + env: + USERNAME: ${{ needs.configure.outputs.username }} timeout-minutes: 40 with: runner_type: 16core-tester-x86 @@ -252,7 +274,7 @@ jobs: needs: - acir-bench - bench-e2e - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + runs-on: ${{ needs.configure.outputs.username }}-x86 steps: - uses: actions/checkout@v4 with: @@ -287,9 +309,9 @@ jobs: AZTEC_BOT_GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} bb-gcc: - needs: [build, changes] - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 - if: needs.changes.outputs.barretenberg-cpp == 'true' + needs: [build, configure] + runs-on: ${{ needs.configure.outputs.username }}-x86 + if: needs.configure.outputs.barretenberg-cpp == 'true' steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -308,14 +330,16 @@ jobs: # ran on own runner for resource reasons (memory x cpu intensive) bb-native-tests: runs-on: ubuntu-20.04 - needs: [build-images, changes] - if: needs.changes.outputs.barretenberg-cpp == 'true' + needs: [build-images, configure] + if: needs.configure.outputs.barretenberg-cpp == 'true' steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } - uses: ./.github/ci-setup-action - name: "Native Prover Tests" uses: ./.github/ensure-tester + env: + USERNAME: ${{ needs.configure.outputs.username }} # note: tester by default times out at 30, would need to delay shutdwon for more timeout-minutes: 30 with: @@ -325,9 +349,9 @@ jobs: scripts/earthly-ci --exec-stats --no-output --no-cache ./barretenberg/cpp+test --hardware_concurrency=32 bb-js-test: - needs: [build, changes] - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 - if: needs.changes.outputs.barretenberg == 'true' + needs: [build, configure] + runs-on: ${{ needs.configure.outputs.username }}-x86 + if: needs.configure.outputs.barretenberg == 'true' steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -340,9 +364,9 @@ jobs: run: earthly-ci --no-output ./+test noir-build-acir-tests: - needs: [build, changes] - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 - if: needs.changes.outputs.barretenberg == 'true' || needs.changes.outputs.noir == 'true' + needs: [build, configure] + runs-on: ${{ needs.configure.outputs.username }}-x86 + if: needs.configure.outputs.barretenberg == 'true' || needs.configure.outputs.noir == 'true' steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -354,9 +378,9 @@ jobs: run: earthly-ci --no-output ./noir/+build-acir-tests bb-acir-tests-bb: - needs: [noir-build-acir-tests, changes] - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 - if: needs.changes.outputs.barretenberg == 'true' || needs.changes.outputs.noir == 'true' + needs: [noir-build-acir-tests, configure] + runs-on: ${{ needs.configure.outputs.username }}-x86 + if: needs.configure.outputs.barretenberg == 'true' || needs.configure.outputs.noir == 'true' steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -369,9 +393,9 @@ jobs: run: earthly-ci --no-output ./+barretenberg-acir-tests-bb bb-acir-tests-sol: - needs: [noir-build-acir-tests, changes] - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 - if: needs.changes.outputs.barretenberg == 'true' || needs.changes.outputs.noir == 'true' + needs: [noir-build-acir-tests, configure] + runs-on: ${{ needs.configure.outputs.username }}-x86 + if: needs.configure.outputs.barretenberg == 'true' || needs.configure.outputs.noir == 'true' steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -384,9 +408,9 @@ jobs: run: earthly-ci --no-output ./+barretenberg-acir-tests-sol bb-acir-tests-sol-honk: - needs: [noir-build-acir-tests, changes] - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 - if: needs.changes.outputs.barretenberg == 'true' || needs.changes.outputs.noir == 'true' + needs: [noir-build-acir-tests, configure] + runs-on: ${{ needs.configure.outputs.username }}-x86 + if: needs.configure.outputs.barretenberg == 'true' || needs.configure.outputs.noir == 'true' steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -399,9 +423,9 @@ jobs: run: earthly-ci --no-output ./+barretenberg-acir-tests-sol-honk bb-acir-tests-bb-js: - needs: [noir-build-acir-tests, changes] - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 - if: needs.changes.outputs.barretenberg == 'true' || needs.changes.outputs.noir == 'true' + needs: [noir-build-acir-tests, configure] + runs-on: ${{ needs.configure.outputs.username }}-x86 + if: needs.configure.outputs.barretenberg == 'true' || needs.configure.outputs.noir == 'true' steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -414,9 +438,9 @@ jobs: run: earthly-ci --no-output ./+barretenberg-acir-tests-bb.js noir-format: - needs: [build, changes] - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 - if: needs.changes.outputs.noir == 'true' || needs.changes.outputs.noir-projects == 'true' + needs: [build, configure] + runs-on: ${{ needs.configure.outputs.username }}-x86 + if: needs.configure.outputs.noir == 'true' || needs.configure.outputs.noir-projects == 'true' steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -434,9 +458,9 @@ jobs: earthly-ci --no-output ./+format noir-test: - needs: [build, changes] - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 - if: needs.changes.outputs.noir == 'true' + needs: [build, configure] + runs-on: ${{ needs.configure.outputs.username }}-x86 + if: needs.configure.outputs.noir == 'true' steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -447,9 +471,9 @@ jobs: run: earthly-ci --no-output ./noir+test noir-examples: - needs: [build, changes] - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 - if: needs.changes.outputs.barretenberg == 'true' || needs.changes.outputs.noir == 'true' + needs: [build, configure] + runs-on: ${{ needs.configure.outputs.username }}-x86 + if: needs.configure.outputs.barretenberg == 'true' || needs.configure.outputs.noir == 'true' steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -460,9 +484,9 @@ jobs: run: earthly-ci --no-output ./noir+examples noir-packages-test: - needs: [build, changes] - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 - if: needs.changes.outputs.barretenberg == 'true' || needs.changes.outputs.noir == 'true' + needs: [build, configure] + runs-on: ${{ needs.configure.outputs.username }}-x86 + if: needs.configure.outputs.barretenberg == 'true' || needs.configure.outputs.noir == 'true' steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -474,9 +498,9 @@ jobs: run: earthly-ci --no-output ./noir+packages-test noir-projects: - needs: [build, changes] - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 - if: needs.changes.outputs.barretenberg == 'true' || needs.changes.outputs.noir == 'true' || needs.changes.outputs.noir-projects == 'true' || needs.changes.outputs.txe == 'true' + needs: [build, configure] + runs-on: ${{ needs.configure.outputs.username }}-x86 + if: needs.configure.outputs.barretenberg == 'true' || needs.configure.outputs.noir == 'true' || needs.configure.outputs.noir-projects == 'true' || needs.configure.outputs.txe == 'true' steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -489,9 +513,9 @@ jobs: earthly-ci --no-output ./noir-projects/+test --RAYON_NUM_THREADS=$(nproc) avm-format: - needs: [build, changes] - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 - if: needs.changes.outputs.avm-transpiler == 'true' || needs.changes.outputs.noir == 'true' + needs: [build, configure] + runs-on: ${{ needs.configure.outputs.username }}-x86 + if: needs.configure.outputs.avm-transpiler == 'true' || needs.configure.outputs.noir == 'true' steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -504,8 +528,8 @@ jobs: run: earthly-ci --no-output ./+format yarn-project-formatting: - needs: build - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + needs: [build, configure] + runs-on: ${{ needs.configure.outputs.username }}-x86 steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -517,8 +541,8 @@ jobs: run: earthly-ci --no-output ./yarn-project/+format-check yarn-project-test: - needs: build - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + needs: [build, configure] + runs-on: ${{ needs.configure.outputs.username }}-x86 steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -530,8 +554,8 @@ jobs: run: earthly-ci --no-output ./yarn-project/+test prover-client-test: - needs: build - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + needs: [build, configure] + runs-on: ${{ needs.configure.outputs.username }}-x86 steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -543,8 +567,8 @@ jobs: run: earthly-ci --no-output ./yarn-project/+prover-client-test network-test-fake-proofs: - needs: build - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + needs: [build, configure] + runs-on: ${{ needs.configure.outputs.username }}-x86 steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -556,22 +580,22 @@ jobs: run: earthly-ci --no-output ./yarn-project/+network-test-fake-proofs l1-contracts-test: - needs: [build, changes] - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 - if: needs.changes.outputs.l1-contracts == 'true' + needs: [build, configure] + runs-on: ${{ needs.configure.outputs.username }}-x86 + if: needs.configure.outputs.l1-contracts == 'true' steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } - uses: ./.github/ci-setup-action with: - concurrency_key: l1-contracts-${{ github.event.pull_request.user.login || github.actor }}-x86 + concurrency_key: l1-contracts-${{ needs.configure.outputs.username }}-x86 - name: "Test l1 contracts" run: earthly-ci --no-output ./l1-contracts+test docs-preview: - needs: [build, changes] - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 - if: needs.changes.outputs.non-barretenberg-cpp == 'true' + needs: [build, configure] + runs-on: ${{ needs.configure.outputs.username }}-x86 + if: needs.configure.outputs.non-barretenberg-cpp == 'true' steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } @@ -590,14 +614,16 @@ jobs: bb-bench: runs-on: ubuntu-20.04 - needs: [build, changes] - if: needs.changes.outputs.barretenberg-cpp == 'true' + needs: [build, configure] + if: needs.configure.outputs.barretenberg-cpp == 'true' steps: - uses: actions/checkout@v4 with: { ref: "${{ env.GIT_COMMIT }}" } - uses: ./.github/ci-setup-action - name: Build Bench Binaries uses: ./.github/ensure-builder + env: + USERNAME: ${{ needs.configure.outputs.username }} with: runner_type: builder-x86 run: | @@ -606,13 +632,13 @@ jobs: scripts/earthly-ci --push ./barretenberg/cpp/+bench-binaries - name: Run Bench uses: ./.github/ensure-tester + env: + USERNAME: ${{ needs.configure.outputs.username }} timeout-minutes: 40 with: runner_type: 16core-tester-x86 run: | scripts/earthly-ci --artifact ./barretenberg/cpp/+bench/bench.json --bench_mode=cache - - # TODO(AD): revisit this model as it's currently fiddly to get these artifacts off of the 'tester' - name: Copy Bench.json run: copy_from_tester /home/ubuntu/run-${{ env.RUN_ID }}/barretenberg/cpp/bench.json bench.json # Utilize github-action-benchmark to automatically update the plots at @@ -637,16 +663,15 @@ jobs: max-items-in-chart: 50 boxes: - needs: build - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + needs: [build, configure] + runs-on: ${{ needs.configure.outputs.username }}-x86 if: github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'boxes') || contains(github.event.pull_request.labels.*.name, 'e2e-all') steps: - uses: actions/checkout@v4 with: { ref: "${{ github.event.pull_request.head.sha }}" } - uses: ./.github/ci-setup-action with: - # must be globally unique for build x runner - concurrency_key: boxes-${{ github.event.pull_request.user.login || github.actor }}-build + concurrency_key: boxes - name: Build working-directory: ./boxes timeout-minutes: 40 @@ -654,7 +679,7 @@ jobs: boxes-test: needs: boxes - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + runs-on: ${{ needs.configure.outputs.username }}-x86 strategy: fail-fast: false matrix: @@ -668,16 +693,16 @@ jobs: - uses: ./.github/ci-setup-action with: # must be globally unique for build x runner - concurrency_key: boxes-${{ github.event.pull_request.user.login || github.actor }}-x86-${{ matrix.box }}-${{ matrix.browser }} + concurrency_key: boxes-${{ needs.configure.outputs.username }}-x86-${{ matrix.box }}-${{ matrix.browser }} - name: Box test working-directory: ./boxes timeout-minutes: 40 run: earthly-ci -P --no-output +test --box=${{ matrix.box }} --browser=${{ matrix.browser }} --mode=cache protocol-circuits-gates-report: - needs: [build, changes] - if: needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true' - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + needs: [build, configure] + if: needs.configure.outputs.non-docs == 'true' && needs.configure.outputs.non-misc-ci == 'true' + runs-on: ${{ needs.configure.outputs.username }}-x86 permissions: pull-requests: write steps: @@ -711,9 +736,9 @@ jobs: message: ${{ steps.gates_diff.outputs.markdown }} public-functions-size-report: - needs: [build, changes] - if: needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true' - runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86 + needs: [build, configure] + if: needs.configure.outputs.non-docs == 'true' && needs.configure.outputs.non-misc-ci == 'true' + runs-on: ${{ needs.configure.outputs.username }}-x86 permissions: pull-requests: write steps: @@ -756,7 +781,7 @@ jobs: needs: # must be kept in sync with rerun-check - setup - - changes + - configure - build - e2e # - bench-e2e # non-blocking @@ -812,7 +837,7 @@ jobs: needs: # must be kept in sync with merge-check - setup - - changes + - configure - build - e2e # - bench-e2e # non-blocking From 0b11e9c187dc5508917d8adef3de02acad6f4c56 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 12 Oct 2024 00:22:13 +0000 Subject: [PATCH 2/5] revert --- .github/workflows/ci.yml | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96086f3e9c3..8b44e6885df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,21 +65,30 @@ jobs: non-misc-ci: ${{ steps.filter.outputs.non-misc-ci }} non-barretenberg-cpp: ${{ steps.filter.outputs.non-barretenberg-cpp }} steps: + - uses: actions/checkout@v4 + with: { ref: "${{ env.GIT_COMMIT }}" } - name: Compute Username id: compute_username shell: bash + env: + REPO: "${{ github.repository }}" + BRANCH: "${{ github.ref_name }}" run: | if [ "${{ github.event_name }}" == "pull_request" ]; then echo "username=${{ github.event.pull_request.user.login }}" >> $GITHUB_OUTPUT else - GIT_HASH="${{ github.sha }}" - GIT_HASH_LAST8=${GIT_HASH: -8} - GIT_HASH_LAST8_DEC=$(printf "%d" 0x$GIT_HASH_LAST8) - GIT_HASH_MODULO_8=$((GIT_HASH_LAST8_DEC % 8)) - echo "username=master-${GIT_HASH_MODULO_8}" >> $GITHUB_OUTPUT + # Fetch the total number of commits on the current branch + # We use gh api to avoid a full history fetch + COMMIT_COUNT=$(gh api -H "Accept: application/vnd.github.v3+json" \ + "/repos/${REPO}/commits?per_page=1&sha=${BRANCH}" \ + -q '. | length') + + # Calculate runner index based on modulo operation + COMMIT_MODULO_8=$((COMMIT_COUNT % 8)) + + # Output the runner-specific username + echo "username=master-${COMMIT_MODULO_8}" >> $GITHUB_OUTPUT fi - - uses: actions/checkout@v4 - with: { ref: "${{ env.GIT_COMMIT }}" } - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 id: filter with: From 16c05ace49a9f353b5d64102ec169234fa14bf34 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 12 Oct 2024 13:58:04 +0100 Subject: [PATCH 3/5] fix null bench runs --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b44e6885df..a2140d2c61b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -219,7 +219,7 @@ jobs: # all the benchmarking end-to-end integration tests for aztec (not required to merge) bench-e2e: needs: [build, configure] - if: (needs.configure.outputs.non-docs == 'true' && needs.configure.outputs.non-misc-ci == 'true' && needs.configure.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'bench') + if: needs.build.outputs.bench_list != '[]' && ((needs.configure.outputs.non-docs == 'true' && needs.configure.outputs.non-misc-ci == 'true' && needs.configure.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'bench')) runs-on: ubuntu-20.04 strategy: fail-fast: false From a9c1788c7cf9250bf617c67950659043d0849d59 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 12 Oct 2024 15:24:28 +0100 Subject: [PATCH 4/5] fix: boxes --- .github/workflows/ci.yml | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a2140d2c61b..c35d1606318 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,14 +35,6 @@ env: WAIT_FOR_RUNNERS: false jobs: - setup: - if: github.event.pull_request.draft == false - uses: ./.github/workflows/setup-runner.yml - with: - username: ${{ github.event.pull_request.user.login || github.actor }} - runner_type: builder-x86 - secrets: inherit - configure: runs-on: ubuntu-20.04 if: github.event.pull_request.draft == false @@ -75,19 +67,15 @@ jobs: BRANCH: "${{ github.ref_name }}" run: | if [ "${{ github.event_name }}" == "pull_request" ]; then + echo "username=${{ github.event.pull_request.user.login }}" echo "username=${{ github.event.pull_request.user.login }}" >> $GITHUB_OUTPUT else - # Fetch the total number of commits on the current branch - # We use gh api to avoid a full history fetch - COMMIT_COUNT=$(gh api -H "Accept: application/vnd.github.v3+json" \ - "/repos/${REPO}/commits?per_page=1&sha=${BRANCH}" \ - -q '. | length') - - # Calculate runner index based on modulo operation - COMMIT_MODULO_8=$((COMMIT_COUNT % 8)) - - # Output the runner-specific username - echo "username=master-${COMMIT_MODULO_8}" >> $GITHUB_OUTPUT + GIT_HASH="${{ github.sha }}" + GIT_HASH_LAST8=${GIT_HASH: -8} + GIT_HASH_LAST8_DEC=$(printf "%d" 0x$GIT_HASH_LAST8) + GIT_HASH_MODULO_8=$((GIT_HASH_LAST8_DEC % 8)) + echo "username=master-${GIT_HASH_MODULO_8}" + echo "username=master-${GIT_HASH_MODULO_8}" >> $GITHUB_OUTPUT fi - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 id: filter @@ -124,6 +112,15 @@ jobs: # Always rebuild when this file changes - .github/workflows/ci.yml + setup: + needs: [configure] + if: github.event.pull_request.draft == false + uses: ./.github/workflows/setup-runner.yml + with: + username: ${{ needs.configure.outputs.username }}-x86 + runner_type: builder-x86 + secrets: inherit + build-images: needs: [setup, configure] # Note we don't but the 'if' here as that would also apply to dependent jobs, instead we just copy it into each step @@ -687,7 +684,7 @@ jobs: run: earthly-ci +export-boxes boxes-test: - needs: boxes + needs: [boxes, configure] runs-on: ${{ needs.configure.outputs.username }}-x86 strategy: fail-fast: false From a7ab76bdd688bb137c9e7891c096f7082e1939d3 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 12 Oct 2024 16:31:20 +0100 Subject: [PATCH 5/5] Update ci.yml --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c35d1606318..89f402aba2a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -280,6 +280,7 @@ jobs: needs: - acir-bench - bench-e2e + - configure runs-on: ${{ needs.configure.outputs.username }}-x86 steps: - uses: actions/checkout@v4