docker repo overhaul (#545)

Closes #539 This PR overhauls the `docker` according to #539. Multi-arch builds inspired from rapidsai/miniforge-cuda#31 Note: this PR merges into `branch-23.08-overhaul` instead of `branch-23.08`. `branch-23.08-overhaul` was manually created to remove the vast majority of the repo. Merging this PR into that branch shows more obvious changes instead of a [long list of deleted files](branch-23.08...branch-23.08-overhaul). To-do: - [x] Handle uploading nightlies with alpha version tag - [x] `main` builds should clean up `condarc` - [x] Add tests - [x] Test all notebooks (not just cudf) Follow up PRs: - Dockerhub/NGC readme updater - Enable nightly tests - Remove required `up-to-date` status check - Merge `branch-23.08-overhaul` into `branch-23.08` - Add style checking (#545 (comment)) Authors: - Ray Douglass (https://github.com/raydouglass)
rapidsai · Jul 25, 2023 · f39c648 · f39c648
1 parent 4eb8bc6
commit f39c648
Show file tree

Hide file tree

Showing 20 changed files with 976 additions and 203 deletions.
diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml
@@ -0,0 +1,106 @@
+name: Build and push image variant
+
+on:
+  workflow_call:
+    inputs:
+      ARCHES:
+        required: true
+        type: string
+      CUDA_VER:
+        required: true
+        type: string
+      LINUX_VER:
+        required: true
+        type: string
+      PYTHON_VER:
+        required: true
+        type: string
+      RAPIDS_VER:
+        required: true
+        type: string
+      DASK_SQL_VER:
+        required: true
+        type: string
+      BASE_TAG:
+        required: true
+        type: string
+      NOTEBOOKS_TAG:
+        required: true
+        type: string
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        ARCH: ${{ fromJSON(inputs.ARCHES) }}
+        CUDA_VER: ["${{ inputs.CUDA_VER }}"]
+        LINUX_VER: ["${{ inputs.LINUX_VER }}"]
+        PYTHON_VER: ["${{ inputs.PYTHON_VER }}"]
+        RAPIDS_VER: ["${{ inputs.RAPIDS_VER }}"]
+        DASK_SQL_VER: ["${{ inputs.DASK_SQL_VER }}"]
+      fail-fast: false
+    runs-on: "linux-${{ matrix.ARCH }}-cpu4"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Install gha-tools
+        run: |
+          mkdir -p /tmp/gha-tools
+          curl -s -L 'https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz' | tar -xz -C /tmp/gha-tools
+          echo "/tmp/gha-tools" >> "${GITHUB_PATH}"
+      - name: Clean up condarc for release builds
+        run: |
+          GIT_DESCRIBE_TAG="$(git describe --tags --abbrev=0)"
+          GIT_DESCRIBE_TAG="${GIT_DESCRIBE_TAG:1}" # remove leading 'v'
+          if [[ ! $GIT_DESCRIBE_TAG =~ [a-z] ]]; then
+            rapids-logger "Most recent tag is for release, adding `rapidsai` channel and removing `rapidsai-nightly` and `dask/label/dev` channels."
+            sed -i 's|rapidsai-nightly|rapidsai|;\|dask/label/dev|d' context/condarc
+          else
+            rapids-logger "Most recent tag is an alpha. Build will use nightly channels."
+          fi
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }}
+          password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}
+      - name: Set up Docker Context for Buildx
+        id: buildx-context
+        run: |
+          docker context create builders
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          driver: docker
+          endpoint: builders
+      - name: Build base image
+        uses: docker/build-push-action@v4
+        with:
+          context: context
+          file: Dockerfile
+          target: base
+          push: true
+          pull: true
+          build-args: |
+            CUDA_VER=${{ inputs.CUDA_VER }}
+            LINUX_VER=${{ inputs.LINUX_VER }}
+            PYTHON_VER=${{ inputs.PYTHON_VER }}
+            RAPIDS_VER=${{ inputs.RAPIDS_VER }}
+            DASK_SQL_VER=${{ inputs.DASK_SQL_VER }}
+          tags: ${{ inputs.BASE_TAG }}-${{ matrix.ARCH }}
+      - name: Build notebooks image
+        uses: docker/build-push-action@v4
+        with:
+          context: context
+          file: Dockerfile
+          target: notebooks
+          push: true
+          pull: true
+          build-args: |
+            CUDA_VER=${{ inputs.CUDA_VER }}
+            LINUX_VER=${{ inputs.LINUX_VER }}
+            PYTHON_VER=${{ inputs.PYTHON_VER }}
+            RAPIDS_VER=${{ inputs.RAPIDS_VER }}
+            DASK_SQL_VER=${{ inputs.DASK_SQL_VER }}
+          tags: ${{ inputs.NOTEBOOKS_TAG }}-${{ matrix.ARCH }}
diff --git a/.github/workflows/build-test-publish-images.yml b/.github/workflows/build-test-publish-images.yml
@@ -0,0 +1,246 @@
+name: build and publish imgs workflow
+
+on:
+  workflow_call:
+    inputs:
+      build_type:
+        required: true
+        type: string
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  actions: read
+  checks: none
+  contents: read
+  deployments: none
+  discussions: none
+  id-token: write
+  issues: none
+  packages: read
+  pages: none
+  pull-requests: read
+  repository-projects: none
+  security-events: none
+  statuses: none
+
+jobs:
+  compute-matrix:
+    runs-on: ubuntu-latest
+    container:
+      image: rapidsai/ci:latest
+    outputs:
+      BASE_IMAGE_REPO: ${{ steps.compute-image-repo.outputs.BASE_IMAGE_REPO }}
+      NOTEBOOKS_IMAGE_REPO: ${{ steps.compute-image-repo.outputs.NOTEBOOKS_IMAGE_REPO }}
+      MATRIX: ${{ steps.compute-matrix.outputs.MATRIX }}
+      TEST_MATRIX: ${{ steps.compute-test-matrix.outputs.TEST_MATRIX }}
+      BASE_TAG_PREFIX: ${{ steps.compute-tag-prefix.outputs.BASE_TAG_PREFIX }}
+      NOTEBOOKS_TAG_PREFIX: ${{ steps.compute-tag-prefix.outputs.NOTEBOOKS_TAG_PREFIX }}
+      RAPIDS_VER: ${{ steps.compute-rapids-ver.outputs.RAPIDS_VER }}
+      ALPHA_TAG: ${{ steps.compute-rapids-ver.outputs.ALPHA_TAG }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Compute matrix
+        id: compute-matrix
+        run: |
+          MATRIX=$(ci/compute-matrix.sh)
+          echo "MATRIX=${MATRIX}" | tee -a ${GITHUB_OUTPUT}
+      - name: Compute tag prefix
+        id: compute-tag-prefix
+        run: |
+          TAG_PREFIX=""
+          if [ "${{ inputs.build_type }}" = "pull-request" ]; then
+            pr_num="${GITHUB_REF_NAME##*/}"
+            BASE_TAG_PREFIX="rapidsai-${pr_num}-"
+            NOTEBOOKS_TAG_PREFIX="rapidsai-notebooks-${pr_num}-"
+          fi
+          echo "BASE_TAG_PREFIX=${BASE_TAG_PREFIX}" | tee -a ${GITHUB_OUTPUT}
+          echo "NOTEBOOKS_TAG_PREFIX=${NOTEBOOKS_TAG_PREFIX}" | tee -a ${GITHUB_OUTPUT}
+      - name: Compute image repo
+        id: compute-image-repo
+        run: |
+          base_repo="rapidsai"
+          notebooks_repo="rapidsai-notebooks"
+          if [ "${{ inputs.build_type }}" = "pull-request" ]; then
+            base_repo="staging"
+            notebooks_repo="staging"
+          fi
+          echo "BASE_IMAGE_REPO=${base_repo}" | tee -a ${GITHUB_OUTPUT}
+          echo "NOTEBOOKS_IMAGE_REPO=${notebooks_repo}" | tee -a ${GITHUB_OUTPUT}
+      - name: Compute RAPIDS_VER
+        id: compute-rapids-ver
+        run: |
+          GIT_DESCRIBE_TAG="$(git describe --tags --abbrev=0)"
+          GIT_DESCRIBE_TAG="${GIT_DESCRIBE_TAG:1}" # remove leading 'v'
+          ALPHA_TAG=""
+          if [[ $GIT_DESCRIBE_TAG =~ [a-z] ]]; then
+            rapids-logger "Most recent tag is an alpha tag"
+            ALPHA_TAG="a"
+          fi
+          RAPIDS_VER="$(echo $GIT_DESCRIBE_TAG | awk 'BEGIN{FS=OFS="."} NF--')" # Convert full tag to YY.MM
+
+          echo "RAPIDS_VER=${RAPIDS_VER}" | tee -a ${GITHUB_OUTPUT}
+          echo "ALPHA_TAG=${ALPHA_TAG}" | tee -a ${GITHUB_OUTPUT}
+      - name: Compute test matrix
+        id: compute-test-matrix
+        run: |
+          TEST_MATRIX=$(yq '.${{ inputs.build_type }}' matrix-test.yaml)
+          export TEST_MATRIX
+
+          echo "TEST_MATRIX=$(yq -n -o json 'env(TEST_MATRIX)' | jq -c '{include: .}')" | tee --append "${GITHUB_OUTPUT}"
+  build:
+    needs: compute-matrix
+    strategy:
+      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
+      fail-fast: false
+    secrets: inherit
+    uses: ./.github/workflows/build-image.yml
+    with:
+      ARCHES: ${{ toJSON(matrix.ARCHES) }}
+      CUDA_VER: ${{ matrix.CUDA_VER }}
+      LINUX_VER: ${{ matrix.LINUX_VER }}
+      PYTHON_VER: ${{ matrix.PYTHON_VER }}
+      RAPIDS_VER: ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
+      DASK_SQL_VER: ${{ matrix.DASK_SQL_VER }}
+      BASE_TAG:
+        "rapidsai/${{ needs.compute-matrix.outputs.BASE_IMAGE_REPO }}:\
+        ${{ needs.compute-matrix.outputs.BASE_TAG_PREFIX }}\
+        ${{ needs.compute-matrix.outputs.RAPIDS_VER }}\
+        ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-\
+        cuda${{ matrix.CUDA_VER }}-\
+        py${{ matrix.PYTHON_VER }}"
+      NOTEBOOKS_TAG:
+        "rapidsai/${{ needs.compute-matrix.outputs.NOTEBOOKS_IMAGE_REPO }}:\
+        ${{ needs.compute-matrix.outputs.NOTEBOOKS_TAG_PREFIX }}\
+        ${{ needs.compute-matrix.outputs.RAPIDS_VER }}\
+        ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-\
+        cuda${{ matrix.CUDA_VER }}-\
+        py${{ matrix.PYTHON_VER }}"
+  test:
+    needs: [build, compute-matrix]
+    # TODO: nightly tests
+    if: inputs.build_type == 'pull-request'
+    strategy:
+      matrix: ${{ fromJSON(needs.compute-matrix.outputs.TEST_MATRIX) }}
+      fail-fast: false
+    secrets: inherit
+    uses: ./.github/workflows/test-notebooks.yml
+    with:
+      BUILD_TYPE: ${{ inputs.build_type }}
+      ARCH: ${{ matrix.ARCH }}
+      GPU: ${{ matrix.GPU }}
+      DRIVER: ${{ matrix.DRIVER }}
+      CUDA_VER: ${{ matrix.CUDA_VER }}
+      PYTHON_VER: ${{ matrix.PYTHON_VER }}
+      NOTEBOOKS_TAG:
+        "rapidsai/${{ needs.compute-matrix.outputs.NOTEBOOKS_IMAGE_REPO }}:\
+        ${{ needs.compute-matrix.outputs.NOTEBOOKS_TAG_PREFIX }}\
+        ${{ needs.compute-matrix.outputs.RAPIDS_VER }}\
+        ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-\
+        cuda${{ matrix.CUDA_VER }}-\
+        py${{ matrix.PYTHON_VER }}-\
+        ${{ matrix.ARCH }}"
+  build-multiarch-manifest:
+    if: inputs.build_type == 'branch'
+    needs: [build, test, compute-matrix]
+    strategy:
+      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }}
+          password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}
+      - name: Create multiarch manifest
+        run: |
+          base_source_tags=()
+          notebooks_source_tags=()
+
+          base_tag_array=(
+            rapidsai/${{ needs.compute-matrix.outputs.BASE_IMAGE_REPO }}:
+            ${{ needs.compute-matrix.outputs.BASE_TAG_PREFIX }}
+            ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
+            ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-
+            cuda${{ matrix.CUDA_VER }}-
+            py${{ matrix.PYTHON_VER }}
+          )
+          base_tag="$(printf %s "${base_tag_array[@]}" $'\n')" # Converts array to string w/o spaces
+
+          notebooks_tag_array=(
+            rapidsai/${{ needs.compute-matrix.outputs.NOTEBOOKS_IMAGE_REPO }}:
+            ${{ needs.compute-matrix.outputs.NOTEBOOKS_TAG_PREFIX }}
+            ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
+            ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-
+            cuda${{ matrix.CUDA_VER }}-
+            py${{ matrix.PYTHON_VER }}
+          )
+          notebooks_tag="$(printf %s "${notebooks_tag_array[@]}" $'\n')" # Converts array to string w/o spaces
+
+          for arch in $(echo '${{ toJSON(matrix.ARCHES) }}' | jq .[] -r); do
+            base_source_tags+=("${base_tag}-${arch}")
+            notebooks_source_tags+=("${notebooks_tag}-${arch}")
+          done
+
+          docker manifest create ${base_tag} ${base_source_tags[@]}
+          docker manifest push ${base_tag}
+          docker manifest create ${notebooks_tag} ${notebooks_source_tags[@]}
+          docker manifest push ${notebooks_tag}
+  delete-temp-images:
+    if: always()
+    needs: [compute-matrix, build-multiarch-manifest]
+    strategy:
+      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Remove temporary images
+        run: |
+          HUB_TOKEN=$(
+            curl -s -H "Content-Type: application/json" \
+              -X POST \
+              -d "{\"username\": \"${{ secrets.GPUCIBOT_DOCKERHUB_USER }}\", \"password\": \"${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}\"}" \
+              https://hub.docker.com/v2/users/login/ | jq -r .token \
+          )
+          echo "::add-mask::${HUB_TOKEN}"
+
+          org="rapidsai"
+          base_repo="${{ needs.compute-matrix.outputs.BASE_IMAGE_REPO }}"
+          base_tag_array=(
+            ${{ needs.compute-matrix.outputs.BASE_TAG_PREFIX }}
+            ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
+            ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-
+            cuda${{ matrix.CUDA_VER }}-
+            py${{ matrix.PYTHON_VER }}
+          )
+          base_tag="$(printf %s "${base_tag_array[@]}" $'\n')" # Converts array to string w/o spaces
+
+
+          notebooks_repo="${{ needs.compute-matrix.outputs.NOTEBOOKS_IMAGE_REPO }}"
+          notebooks_tag_array=(
+            ${{ needs.compute-matrix.outputs.NOTEBOOKS_TAG_PREFIX }}
+            ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
+            ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-
+            cuda${{ matrix.CUDA_VER }}-
+            py${{ matrix.PYTHON_VER }}
+          )
+          notebooks_tag="$(printf %s "${notebooks_tag_array[@]}" $'\n')" # Converts array to string w/o spaces
+
+          for arch in $(echo '${{ toJSON(matrix.ARCHES) }}' | jq .[] -r); do
+            curl -i -X DELETE \
+              -H "Accept: application/json" \
+              -H "Authorization: JWT $HUB_TOKEN" \
+              "https://hub.docker.com/v2/repositories/$org/$base_repo/tags/$base_tag-$arch/"
+            curl -i -X DELETE \
+              -H "Accept: application/json" \
+              -H "Authorization: JWT $HUB_TOKEN" \
+              "https://hub.docker.com/v2/repositories/$org/$notebooks_repo/tags/$notebooks_tag-$arch/"
+          done
diff --git a/.github/workflows/dockerhub-readme-nightly.yml b/.github/workflows/dockerhub-readme-nightly.yml