Use buildx docker-container driver for publishing normalization conta…

…iners (#13693) * explicitly use buildx for connector publish * ... and use it * use docker-container driver * use `driver docker-container` only for normalization * Update tools/integrations/manage.sh Co-authored-by: Edward Gao <[email protected]> Co-authored-by: Edward Gao <[email protected]>
airbytehq · Jun 10, 2022 · 6c8cae2 · 6c8cae2
1 parent 866fb52
commit 6c8cae2
Showing 1 changed file with 7 additions and 0 deletions.
diff --git a/tools/integrations/manage.sh b/tools/integrations/manage.sh
@@ -245,6 +245,10 @@ cmd_publish() {
     echo "Publishing normalization images (version: $versioned_image)"
     GIT_REVISION=$(git rev-parse HEAD)
 
+    # We use a buildx docker container when building multi-stage builds from one docker compose file
+    # This works because all the images depend only on already public images
+    docker buildx create --name connector-buildx --driver docker-container --use
+
     # Note: "buildx bake" needs to be run within the directory
     local original_pwd=$PWD
     cd airbyte-integrations/bases/base-normalization
@@ -259,10 +263,13 @@ cmd_publish() {
       -f docker-compose.build.yaml                               \
       --push
 
+    docker buildx rm connector-buildx
+
     cd $original_pwd
   else
     # We have to go arch-by-arch locally (see https://github.com/docker/buildx/issues/59 for more info) due to our base images (e.g. airbyte-integrations/bases/base-java)
     # Alternative local approach @ https://github.com/docker/buildx/issues/301#issuecomment-755164475
+    # We need to use the regular docker buildx driver (not docker container) because we need this intermediate contaiers to be available for later build steps
 
     for arch in $(echo $build_arch | sed "s/,/ /g")
     do