diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml deleted file mode 100644 index 0f32886c21..0000000000 --- a/.github/workflows/container.yml +++ /dev/null @@ -1,114 +0,0 @@ -name: Publish Docker Container Images -on: - push: - branches: [main] - -env: - REGISTRY: ghcr.io - REPO_NAME: ${{ github.repository }} - -permissions: - contents: read - packages: write - -jobs: - - # ------------------------------------------------------------------------------------------ - # To be decided: Script-Deploy or Dockerfile-Deploy: - # Script: - # + Separates the golang build from the corso build. - # - Haven't figured out multiplatform builds yet. - # - Doesn't cache, always takes 10-15 minutes per build in the matrix. - # Dockerfile: - # + Once cached, takes <1m to deploy. - # + Multiplatform. - # + Extended features (such as tagging) can be handled by more github actions. - # - When not cached, can take >2 hours to build (at least initially). - # - Currently includes the complete golang:1.18 image. - # ------------------------------------------------------------------------------------------ - - Script-Deploy: - runs-on: ubuntu-latest - defaults: - run: - working-directory: build - strategy: - matrix: - BUILD_ARCH: [amd64, arm64] - BUILD_OS: [linux] - env: - IMAGE_PREFIX: ghcr.io - VERSION_SUFFIX: rolling - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Run build script - run: > - ./build-container.sh - --arch ${{ matrix.BUILD_ARCH }} - --prefix ${{ env.IMAGE_PREFIX }} - --suffix ${{ env.VERSION_SUFFIX }} - - # login step boilerplate from: - # https://docs.github.com/en/packages/managing-github-packages-using-github-actions-workflows/publishing-and-installing-a-package-with-github-actions#upgrading-a-workflow-that-accesses-ghcrio - - name: Log in to registry - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin - - - name: Push image - env: - IMAGE_ID: ${{ env.IMAGE_PREFIX }}/alcionai/corso - VERSION: ${{ matrix.BUILD_OS }}-${{ matrix.BUILD_ARCH }}-${{ env.VERSION_SUFFIX }} - run: | - docker images -a - docker push ${{ env.IMAGE_ID }}:${{ env.VERSION }} - - Dockerfile-Deploy: - runs-on: ubuntu-latest - env: - TARGETOS: linux - TARGETARCH: arm64 - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - # apparently everyone uses this step - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - # setup Docker buld action - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - - # In case we want to switch to dockerhub - # - name: Login to DockerHub - # uses: docker/login-action@v2 - # with: - # username: ${{ secrets.DOCKERHUB_USERNAME }} - # password: ${{ secrets.DOCKERHUB_TOKEN }} - - # retrieve credentials for ghcr.io - - name: Login to Github Packages - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - # build the image - - name: Build image and push to Docker Hub and GitHub Container Registry - uses: docker/build-push-action@v3 - with: - context: . - file: ./docker/Dockerfile - platforms: linux/amd64,linux/arm64 - push: true - tags: ghcr.io/alcionai/corso:rolling - # use the github cache - cache-from: type=gha - cache-to: type=gha,mode=max - - # check the image digest - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml new file mode 100644 index 0000000000..91aa8533aa --- /dev/null +++ b/.github/workflows/image.yml @@ -0,0 +1,66 @@ +name: Publish Docker Container Images +on: + push: + branches: [main] + +permissions: + contents: read + packages: write + +jobs: + Per-SHA-Image: + runs-on: ubuntu-latest + defaults: + run: + working-directory: build + env: + PLATFORMS: linux/amd64,linux/arm64 + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Build Corso Binaries + run: > + ./build.sh + --platforms ${{ env.PLATFORMS }} + + # - name: Build Corso Binaries Locally + # run: > + # ./multiplatform-binary.sh + # --platforms ${{ env.PLATFORMS }} + + # apparently everyone uses this step + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + # setup Docker buld action + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + + # retrieve credentials for ghcr.io + - name: Login to Github Packages + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # deploy the image + - name: Build image and push to GitHub Container Registry + env: + sha: git_hash=$(git rev-parse --short "$GITHUB_SHA") + uses: docker/build-push-action@v3 + with: + context: . + file: ./build/Dockerfile + platforms: ${{ env.PLATFORMS }} + push: true + tags: ghcr.io/alcionai/corso:${{ env.sha }} + # use the github cache + cache-from: type=gha + cache-to: type=gha,mode=max + + # check the image digest + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/build/Dockerfile b/build/Dockerfile index 4bc2a9060a..d33c313817 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -8,7 +8,9 @@ FROM gcr.io/distroless/base-debian10 WORKDIR / -COPY ./bin/corso ./ +ARG TARGETOS +ARG TARGETARCH +COPY ./bin/${TARGETOS}-${TARGETARCH}/corso ./ USER nonroot:nonroot diff --git a/build/build-container.sh b/build/build-container.sh index 9bc8dab6f7..8b9144ea1e 100755 --- a/build/build-container.sh +++ b/build/build-container.sh @@ -11,6 +11,7 @@ usage() { echo "Flags" echo " -h|--help Help" echo " -a|--arch Set the architecture to the specified value (default: amd64)" + echo " -l|--local Build the corso binary on your local system, rather than a go image" echo " -p|--prefix Prefixes the image name." echo " -s|--suffix Suffixes the version." echo " " @@ -30,6 +31,7 @@ OS=linux ARCH=amd64 IMAGE_NAME_PREFIX= IMAGE_TAG_SUFFIX= +LOCAL= while [ "$#" -gt 0 ] do @@ -42,6 +44,9 @@ do ARCH=$2 shift ;; + -l|--local) + LOCAL=1 + ;; -p|--prefix) IMAGE_NAME_PREFIX=$2 shift @@ -62,6 +67,8 @@ do shift done +TARGETPLATFORM=${OS}/${ARCH} + IMAGE_TAG=${OS}-${ARCH} if [ ! -z "${IMAGE_TAG_SUFFIX}" ]; then IMAGE_TAG=${IMAGE_TAG}-${IMAGE_TAG_SUFFIX} @@ -72,7 +79,11 @@ if [ ! -z "${IMAGE_NAME_PREFIX}" ]; then IMAGE_NAME=${IMAGE_NAME_PREFIX}/${IMAGE_NAME} fi -${SCRIPT_ROOT}/build.sh --arch ${ARCH} +if [ -z "$LOCAL" ]; then + ${SCRIPT_ROOT}/build.sh --platforms "${TARGETPLATFORM}" +else + ${SCRIPT_ROOT}/multiplatform-binary.sh --platforms "${TARGETPLATFORM}" +fi echo "-----" echo "building corso container ${IMAGE_NAME}" @@ -80,8 +91,8 @@ echo "-----" set -x docker buildx build --tag ${IMAGE_NAME} \ - --platform ${OS}/${ARCH} \ - --file ${PROJECT_ROOT}/build/Dockerfile \ + --platform ${TARGETPLATFORM} \ + --file ${PROJECT_ROOT}/build/Dockerfile \ ${PROJECT_ROOT} set +x diff --git a/build/build.sh b/build/build.sh index 4b82ad542c..7d68f6015f 100755 --- a/build/build.sh +++ b/build/build.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash set -e @@ -16,6 +16,7 @@ CORSO_MOD_CACHE=${CORSO_BUILD_PKG_MOD}/cache CORSO_BUILD_ARGS='' +platforms= GOVER=1.18 GOOS=linux GOARCH=amd64 @@ -23,8 +24,8 @@ GOARCH=amd64 while [ "$#" -gt 0 ] do case "$1" in - --arch) - GOARCH=$2 + --platforms) + platforms=$2 shift ;; esac @@ -36,27 +37,39 @@ mkdir -p ${CORSO_BUILD_TMP_CACHE} # temporary directory for caching go modules (needed for fast cross-platform build) mkdir -p ${CORSO_BUILD_TMP_MOD} -echo "-----" -echo "building corso binary for ${GOOS}-${GOARCH}" -echo "-----" - -set -x -docker run --rm \ - --mount type=bind,src=${PROJECT_ROOT},dst=${CORSO_BUILD_CONTAINER} \ - --mount type=bind,src=${CORSO_BUILD_TMP_CACHE},dst=${CORSO_BUILD_TMP_CACHE} \ - --mount type=bind,src=${CORSO_BUILD_TMP_MOD},dst=${CORSO_BUILD_PKG_MOD} \ - --workdir ${CORSO_BUILD_CONTAINER_SRC} \ - --env GOMODCACHE=${CORSO_MOD_CACHE} \ - --env GOCACHE=${CORSO_CACHE} \ - --env GOOS=${GOOS} \ - --env GOARCH=${GOARCH} \ - --entrypoint /usr/local/go/bin/go \ - golang:${GOVER} \ - build ${CORSO_BUILD_ARGS} -set +x - -mkdir -p ${PROJECT_ROOT}/bin -mv ${PROJECT_ROOT}/src/corso ${PROJECT_ROOT}/bin/corso - -echo "-----" -echo "created binary image in ${PROJECT_ROOT}/bin/corso" +if [ -z "$platforms" ]; then + platforms="${GOOS}/${GOARCH}" +fi + +for platform in ${platforms/,/ } +do + IFS='/' read -r -a platform_split <<< "${platform}" + GOOS=${platform_split[0]} + GOARCH=${platform_split[1]} + + echo "-----" + echo "building corso binary for ${GOOS}/${GOARCH}" + echo "-----" + + set -x + docker run --rm \ + --mount type=bind,src=${PROJECT_ROOT},dst=${CORSO_BUILD_CONTAINER} \ + --mount type=bind,src=${CORSO_BUILD_TMP_CACHE},dst=${CORSO_BUILD_TMP_CACHE} \ + --mount type=bind,src=${CORSO_BUILD_TMP_MOD},dst=${CORSO_BUILD_PKG_MOD} \ + --workdir ${CORSO_BUILD_CONTAINER_SRC} \ + --env GOMODCACHE=${CORSO_MOD_CACHE} \ + --env GOCACHE=${CORSO_CACHE} \ + --env GOOS=${GOOS} \ + --env GOARCH=${GOARCH} \ + --entrypoint /usr/local/go/bin/go \ + golang:${GOVER} \ + build ${CORSO_BUILD_ARGS} + set +x + + mkdir -p ${PROJECT_ROOT}/bin/${GOOS}-${GOARCH} + mv ${PROJECT_ROOT}/src/corso ${PROJECT_ROOT}/bin/${GOOS}-${GOARCH}/corso + + echo "-----" + echo "created binary image in ${PROJECT_ROOT}/bin/${GOOS}-${GOARCH}/corso" + echo "-----" +done \ No newline at end of file diff --git a/build/multiplatform-binary.sh b/build/multiplatform-binary.sh new file mode 100755 index 0000000000..2633abad31 --- /dev/null +++ b/build/multiplatform-binary.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +set -e + +SCRIPT_ROOT=$(dirname $(readlink -f $0)) +PROJECT_ROOT=$(dirname ${SCRIPT_ROOT}) + +platforms= +GOVER=1.18 +GOOS=linux +GOARCH=amd64 + +while [ "$#" -gt 0 ] +do + case "$1" in + --platforms) + platforms=$2 + shift + ;; + esac + shift +done + +CORSO_BUILD_ARGS="$@" + +if [ -z "$platforms" ]; then + platforms="${GOOS}/${GOARCH}" +fi + +for platform in ${platforms/,/ } +do + IFS='/' read -r -a platform_split <<< "${platform}" + GOOS=${platform_split[0]} + GOARCH=${platform_split[1]} + + echo "-----" + echo "building corso binary for ${GOOS}/${GOARCH}" + echo "-----" + + OS_ARCH_DIR=${PROJECT_ROOT}/bin/${GOOS}-${GOARCH} + + set -x + + mkdir -p ${OS_ARCH_DIR} + + cd ${PROJECT_ROOT}/src; \ + GOOS=${GOOS} \ + GOARCH=${GOARCH} \ + go build -o ${OS_ARCH_DIR} "$CORSO_BUILD_ARGS" + + set +x + + echo "-----" + echo "created binary ${PROJECT_ROOT}/bin/${GOOS}-${GOARCH}/corso" + echo "-----" +done \ No newline at end of file diff --git a/docker/docker-bake.hcl b/docker/docker-bake.hcl deleted file mode 100644 index 9292418b3b..0000000000 --- a/docker/docker-bake.hcl +++ /dev/null @@ -1,12 +0,0 @@ -// docker-bake.hcl -target "docker-metadata-action" {} - -target "build" { - inherits = ["docker-metadata-action"] - context = "./" - dockerfile = "Dockerfile" - platforms = [ - "linux/amd64", - "linux/arm64", - ] -} \ No newline at end of file