Publish Multi Arch images (ray-project#1716)

davidxia · Dec 8, 2023 · 56b4d14 · 56b4d14
1 parent 22696fe
commit 56b4d14
Show file tree

Hide file tree

Showing 4 changed files with 118 additions and 24 deletions.
diff --git a/.github/workflows/image-release.yaml b/.github/workflows/image-release.yaml
@@ -207,25 +207,52 @@ jobs:
         username: ${{ secrets.DOCKER_USERNAME }}
         password: ${{ secrets.DOCKER_PASSWORD }}
 
-    - name: Push Operator to DockerHub
-      run: |
-        docker push kuberay/operator:${{ steps.vars.outputs.sha_short }};
-        docker image tag kuberay/operator:${{ steps.vars.outputs.sha_short }} kuberay/operator:${{ github.event.inputs.tag }};
-        docker push kuberay/operator:${{ github.event.inputs.tag }}
-
     - name: Log in to Quay.io
       uses: docker/login-action@v2
       with:
         registry: quay.io
         username: ${{ secrets.QUAY_USERNAME }}
         password: ${{ secrets.QUAY_ROBOT_TOKEN }}
 
-    - name: Push Operator to Quay.io
+    # Build operators inside the gh runner vm directly and then copy the go binaries to docker images using the Dockerfile.buildx
+    - name: Build linux/amd64 Operator go binary
+      env:
+        CGO_ENABLED: 1
+        GOOS: linux
+        GOARCH: amd64
+      run: |
+        CGO_ENABLED=$CGO_ENABLED GOOS=$GOOS GOARCH=$GOARCH go build -tags strictfipsruntime -a -o manager-$GOARCH main.go
+      working-directory: ${{env.working-directory}}
+
+    - name: Build linux/arm64 Operator binary
+      env:
+        CC: aarch64-linux-gnu-gcc
+        CGO_ENABLED: 1
+        GOOS: linux
+        GOARCH: arm64
       run: |
-        docker image tag kuberay/operator:${{ steps.vars.outputs.sha_short }} quay.io/kuberay/operator:${{ steps.vars.outputs.sha_short }};
-        docker push quay.io/kuberay/operator:${{ steps.vars.outputs.sha_short }};
-        docker image tag kuberay/operator:${{ steps.vars.outputs.sha_short }} quay.io/kuberay/operator:${{ github.event.inputs.tag }};
-        docker push quay.io/kuberay/operator:${{ github.event.inputs.tag }}
+        sudo apt-get update
+        sudo apt-get install -y gcc-aarch64-linux-gnu libc6-dev-arm64-cross
+        CC=$CC CGO_ENABLED=$CGO_ENABLED GOOS=$GOOS GOARCH=$GOARCH go build -tags strictfipsruntime -a -o manager-$GOARCH main.go
+      working-directory: ${{env.working-directory}}
+
+    - name: Build MultiArch Image
+      uses: docker/build-push-action@v5
+      env:
+        PUSH: true
+        REPO_ORG: kuberay
+        REPO_NAME: operator
+      with:
+        platforms: linux/amd64,linux/arm64
+        context: ${{env.working-directory}}
+        file: ${{env.working-directory}}/Dockerfile.buildx
+        push: ${{PUSH}}
+        provenance: false
+        tags: |
+          quay.io/${{env.REPO_ORG}}/${{env.REPO_NAME}}:${{ steps.vars.outputs.sha_short }}
+          quay.io/${{env.REPO_ORG}}/${{env.REPO_NAME}}:${{ github.event.inputs.tag }}
+          docker.io/${{env.REPO_ORG}}/${{env.REPO_NAME}}:${{ steps.vars.outputs.sha_short }}
+          docker.io/${{env.REPO_ORG}}/${{env.REPO_NAME}}:${{ github.event.inputs.tag }}
 
     - name: Create tag
       uses: actions/github-script@v6

diff --git a/.github/workflows/test-job.yaml b/.github/workflows/test-job.yaml
@@ -347,13 +347,6 @@ jobs:
         password: ${{ secrets.DOCKER_PASSWORD }}
       if: contains(fromJson('["refs/heads/master"]'), github.ref)
 
-    - name: Push Operator to DockerHub
-      run: |
-        docker push kuberay/operator:${{ steps.vars.outputs.sha_short }};
-        docker image tag kuberay/operator:${{ steps.vars.outputs.sha_short }} kuberay/operator:nightly;
-        docker push kuberay/operator:nightly
-      if: contains(fromJson('["refs/heads/master"]'), github.ref)
-
     - name: Log in to Quay.io
       uses: docker/login-action@v2
       with:
@@ -362,13 +355,45 @@ jobs:
         password: ${{ secrets.QUAY_ROBOT_TOKEN }}
       if: contains(fromJson('["refs/heads/master"]'), github.ref)
 
-    - name: Push Operator to Quay.io
+    # Build operators inside the gh runner vm directly and then copy the go binaries to docker images using the Dockerfile.buildx
+    - name: Compile linux/amd64 Operator go binary
+      env:
+        CGO_ENABLED: 1
+        GOOS: linux
+        GOARCH: amd64
       run: |
-        docker image tag kuberay/operator:${{ steps.vars.outputs.sha_short }} quay.io/kuberay/operator:${{ steps.vars.outputs.sha_short }};
-        docker push quay.io/kuberay/operator:${{ steps.vars.outputs.sha_short }};
-        docker image tag kuberay/operator:${{ steps.vars.outputs.sha_short }} quay.io/kuberay/operator:nightly;
-        docker push quay.io/kuberay/operator:nightly
-      if: contains(fromJson('["refs/heads/master"]'), github.ref)
+        CGO_ENABLED=$CGO_ENABLED GOOS=$GOOS GOARCH=$GOARCH go build -tags strictfipsruntime -a -o manager-$GOARCH main.go
+      working-directory: ${{env.working-directory}}
+
+    - name: Cross Compile linux/arm64 Operator binary
+      env:
+        CC: aarch64-linux-gnu-gcc
+        CGO_ENABLED: 1
+        GOOS: linux
+        GOARCH: arm64
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y gcc-aarch64-linux-gnu libc6-dev-arm64-cross
+        CC=$CC CGO_ENABLED=$CGO_ENABLED GOOS=$GOOS GOARCH=$GOARCH go build -tags strictfipsruntime -a -o manager-$GOARCH main.go
+      working-directory: ${{env.working-directory}}
+
+    - name: Build MultiArch Docker Image and Push on Merge
+      uses: docker/build-push-action@v5
+      env:
+        PUSH: ${{contains(fromJson('["refs/heads/master"]'), github.ref)}}
+        REPO_ORG: kuberay
+        REPO_NAME: operator
+      with:
+        platforms: linux/amd64,linux/arm64
+        context: ${{env.working-directory}}
+        file: ${{env.working-directory}}/Dockerfile.buildx
+        push: ${{env.PUSH}}
+        provenance: false
+        tags: |
+          quay.io/${{env.REPO_ORG}}/${{env.REPO_NAME}}:${{ steps.vars.outputs.sha_short }}
+          quay.io/${{env.REPO_ORG}}/${{env.REPO_NAME}}:nightly
+          docker.io/${{env.REPO_ORG}}/${{env.REPO_NAME}}:${{ steps.vars.outputs.sha_short }}
+          docker.io/${{env.REPO_ORG}}/${{env.REPO_NAME}}:nightly
 
   test-compatibility-2_5_0:
     needs:

diff --git a/ray-operator/DEVELOPMENT.md b/ray-operator/DEVELOPMENT.md
@@ -283,3 +283,38 @@ RAY_IMAGE=rayproject/ray:2.8.0 OPERATOR_IMAGE=kuberay/operator:nightly python3 t
 ```
 
 See [KubeRay PR #605](https://github.com/ray-project/kuberay/pull/605) for more details about the test framework.
+
+### Building Multi architecture images locally
+
+Most of image repositories supports multiple architectures container images. When running an image from a device, the docker client automatically pulls the correct the image with a matching architectures. The easiest way to build multi-arch images is to utilize Docker `Buildx` plug-in which allows easily building multi-arch images using Qemu emulation from a single machine. Buildx plugin is readily available when you install the [Docker Desktop](https://docs.docker.com/desktop/) on your machine.
+Verify Buildx installation and make sure it does not return error
+```
+docker buildx version
+```
+Verify the builder instance has a default(with *) DRIVER/ENDPOINT starting with `docker-container` by running:
+```
+docker buildx ls
+```
+You may see something:
+```
+NAME/NODE    DRIVER/ENDPOINT             STATUS  BUILDKIT             PLATFORMS
+sad_brown *  docker-container
+  sad_brown0 unix:///var/run/docker.sock running v0.12.4              linux/amd64, linux/amd64/v2, linux/amd64/v3, linux/amd64/v4, linux/arm64, linux/riscv64, linux/ppc64le, linux/s390x, linux/386, linux/mips64le, linux/mips64, linux/arm/v7, linux/arm/v6
+default      docker
+  default    default                     running v0.11.7+d3e6c1360f6e linux/amd64, linux/amd64/v2, linux/amd64/v3, linux/amd64/v4, linux/386, linux/arm64, linux/riscv64, linux/ppc64le, linux/s390x, linux/mips64le, linux/mips64, linux/arm/v7, linux/arm/v6
+```
+
+If not, create the instance by running:
+```
+docker buildx create --use --bootstrap
+```
+
+Run the following `docker buildx build` command to build and push linux/arm64 and linux/amd64 images(manifests) in a single command:
+```
+cd ray-operator
+docker buildx build --tag quay.io/<my org>/operator:latest --tag docker.io/<my org>/operator:latest --platform linux/amd64,linux/arm64 --push --provenance=false .
+```
+* --platform is a comma separated list of targeted platforms to build.
+* --tag is a remote repo_name:tag to push.
+* --push/--load optionally Push to remote registry or Load into local docker.
+* Some registry such as Quay.io dashboard displays attestation manifests as unknown platforms. Setting --provenance=false to avoid this issue.
diff --git a/ray-operator/Dockerfile.buildx b/ray-operator/Dockerfile.buildx
@@ -0,0 +1,7 @@
+FROM registry.access.redhat.com/ubi9/ubi-minimal:9.3
+ARG TARGETARCH
+WORKDIR /
+COPY ./manager-${TARGETARCH} ./manager
+USER 65532:65532
+
+ENTRYPOINT ["/manager"]