From 368543cc017db5d58a5671a26c25231e7da2f5f8 Mon Sep 17 00:00:00 2001 From: evantahler Date: Fri, 10 Jun 2022 11:49:42 -0700 Subject: [PATCH 1/5] explicitly use buildx for connector publish --- tools/integrations/manage.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/integrations/manage.sh b/tools/integrations/manage.sh index eec015955255..0b47cd97ffb9 100755 --- a/tools/integrations/manage.sh +++ b/tools/integrations/manage.sh @@ -241,6 +241,8 @@ cmd_publish() { # in case curing the build / tests someone this version has been published. _error_if_tag_exists "$versioned_image" + docker buildx create --name connector-buildx --driver docker + if [[ "airbyte/normalization" == "${image_name}" ]]; then echo "Publishing normalization images (version: $versioned_image)" GIT_REVISION=$(git rev-parse HEAD) @@ -296,6 +298,8 @@ cmd_publish() { fi + docker buildx rm connector-buildx + # Checking if the image was successfully registered on DockerHub # see the description of this PR to understand why this is needed https://github.com/airbytehq/airbyte/pull/11654/ sleep 5 From 67321a2f68e31b5956092e459379e07e95b2550a Mon Sep 17 00:00:00 2001 From: evantahler Date: Fri, 10 Jun 2022 11:51:29 -0700 Subject: [PATCH 2/5] ... and use it --- tools/integrations/manage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/integrations/manage.sh b/tools/integrations/manage.sh index 0b47cd97ffb9..b9066b15915c 100755 --- a/tools/integrations/manage.sh +++ b/tools/integrations/manage.sh @@ -241,7 +241,7 @@ cmd_publish() { # in case curing the build / tests someone this version has been published. _error_if_tag_exists "$versioned_image" - docker buildx create --name connector-buildx --driver docker + docker buildx create --name connector-buildx --driver docker --use if [[ "airbyte/normalization" == "${image_name}" ]]; then echo "Publishing normalization images (version: $versioned_image)" From 54b5e99cb13b3ab142ce029649214ec3cd665a50 Mon Sep 17 00:00:00 2001 From: evantahler Date: Fri, 10 Jun 2022 11:55:30 -0700 Subject: [PATCH 3/5] use docker-container driver --- tools/integrations/manage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/integrations/manage.sh b/tools/integrations/manage.sh index b9066b15915c..79ed835f3b72 100755 --- a/tools/integrations/manage.sh +++ b/tools/integrations/manage.sh @@ -241,7 +241,7 @@ cmd_publish() { # in case curing the build / tests someone this version has been published. _error_if_tag_exists "$versioned_image" - docker buildx create --name connector-buildx --driver docker --use + docker buildx create --name connector-buildx --driver docker-container --use if [[ "airbyte/normalization" == "${image_name}" ]]; then echo "Publishing normalization images (version: $versioned_image)" From a27157ddf84f5dc59dd401087db0809a7333b964 Mon Sep 17 00:00:00 2001 From: evantahler Date: Fri, 10 Jun 2022 12:14:50 -0700 Subject: [PATCH 4/5] use `driver docker-container` only for normalization --- tools/integrations/manage.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tools/integrations/manage.sh b/tools/integrations/manage.sh index 79ed835f3b72..432e556e0030 100755 --- a/tools/integrations/manage.sh +++ b/tools/integrations/manage.sh @@ -241,12 +241,14 @@ cmd_publish() { # in case curing the build / tests someone this version has been published. _error_if_tag_exists "$versioned_image" - docker buildx create --name connector-buildx --driver docker-container --use - if [[ "airbyte/normalization" == "${image_name}" ]]; then echo "Publishing normalization images (version: $versioned_image)" GIT_REVISION=$(git rev-parse HEAD) + # We use a buildx docker continer when building multi-stage builds from one docker compose file + # This works because all the images depend only on already public images + docker buildx create --name connector-buildx --driver docker-container --use + # Note: "buildx bake" needs to be run within the directory local original_pwd=$PWD cd airbyte-integrations/bases/base-normalization @@ -261,10 +263,13 @@ cmd_publish() { -f docker-compose.build.yaml \ --push + docker buildx rm connector-buildx + cd $original_pwd else # We have to go arch-by-arch locally (see https://github.com/docker/buildx/issues/59 for more info) due to our base images (e.g. airbyte-integrations/bases/base-java) # Alternative local approach @ https://github.com/docker/buildx/issues/301#issuecomment-755164475 + # We need to use the regular docker buildx driver (not docker container) because we need this intermediate contaiers to be available for later build steps for arch in $(echo $build_arch | sed "s/,/ /g") do @@ -298,8 +303,6 @@ cmd_publish() { fi - docker buildx rm connector-buildx - # Checking if the image was successfully registered on DockerHub # see the description of this PR to understand why this is needed https://github.com/airbytehq/airbyte/pull/11654/ sleep 5 From 41f145e52f511b28d0c48d5f0d1b17253da316dc Mon Sep 17 00:00:00 2001 From: Evan Tahler Date: Fri, 10 Jun 2022 12:24:00 -0700 Subject: [PATCH 5/5] Update tools/integrations/manage.sh Co-authored-by: Edward Gao --- tools/integrations/manage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/integrations/manage.sh b/tools/integrations/manage.sh index 432e556e0030..93deefa4ce4e 100755 --- a/tools/integrations/manage.sh +++ b/tools/integrations/manage.sh @@ -245,7 +245,7 @@ cmd_publish() { echo "Publishing normalization images (version: $versioned_image)" GIT_REVISION=$(git rev-parse HEAD) - # We use a buildx docker continer when building multi-stage builds from one docker compose file + # We use a buildx docker container when building multi-stage builds from one docker compose file # This works because all the images depend only on already public images docker buildx create --name connector-buildx --driver docker-container --use