From cb63334575c1b46e54d44f9b5b4721d3f919a733 Mon Sep 17 00:00:00 2001
From: Jarek Potiuk <jarek@potiuk.com>
Date: Tue, 13 Feb 2024 22:11:23 +0100
Subject: [PATCH] Fix intermittent failures coming from buildkit parallel
 builds

The PROD cachie build often fails (especially in v2-8-test) when it
tries to rebuild the PROD cache in parallel on ARM. There is some weird
inter-buildx problem with it and some people experience it sometimes
as documented in https://github.com/moby/buildkit/issues/2367

Instead of finding the root cause, we change this specific job in
CI to run sequentially. This changes the time it will take to update
cache - but not as much as 4x because the builds do not parallelise
very well anyway. Likely instead of 8m we will get maybe total 15m,
and since this is just cache update after everything else has
completed, it does not matter too much if it runs a little longer.
---
 .github/workflows/ci.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8d501c9d5fcfd5..6cefe50bd77389 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -2448,14 +2448,14 @@ jobs:
       - name: "Copy dist packages to docker-context files"
         run: cp -v --no-preserve=mode,ownership ./dist/*.whl ./docker-context-files
       - name: "Push PROD cache ${{ matrix.python-version }} ${{ matrix.platform }}"
-        run: >
-          breeze prod-image build
-          --builder airflow_cache
-          --install-packages-from-context
-          --run-in-parallel
-          --airflow-constraints-mode constraints-source-providers
-          --prepare-buildx-cache
-          --platform ${{ matrix.platform }}
+        run: |
+          # Do not run parallel builds here as they often fail due to github token expiry issues similar to
+          # those described in https://github.com/moby/buildkit/issues/2367
+          for python in ${{needs.build-info.outputs.python-versions-list-as-string}}; do
+            breeze prod-image build --builder airflow_cache --install-packages-from-context \
+            --airflow-constraints-mode constraints-source-providers --prepare-buildx-cache \
+            --platform ${{ matrix.platform }} --python ${python}
+          done
         env:
           COMMIT_SHA: ${{ github.sha }}
       - name: "Push PROD latest image ${{ matrix.platform }}"