build-test-publish-images.yml

name: build and publish imgs workflow

on:
  workflow_call:
    inputs:
      build_type:
        required: true
        type: string
      run_tests:
        required: false
        default: false
        type: boolean

defaults:
  run:
    shell: bash

permissions:
  actions: read
  checks: none
  contents: read
  deployments: none
  discussions: none
  id-token: write
  issues: none
  packages: read
  pages: none
  pull-requests: read
  repository-projects: none
  security-events: none
  statuses: none

jobs:
  compute-matrix:
    runs-on: ubuntu-latest
    container:
      image: rapidsai/ci-conda:latest
    outputs:
      BASE_IMAGE_REPO: ${{ steps.compute-image-repo.outputs.BASE_IMAGE_REPO }}
      NOTEBOOKS_IMAGE_REPO: ${{ steps.compute-image-repo.outputs.NOTEBOOKS_IMAGE_REPO }}
      RAFT_ANN_BENCH_IMAGE_REPO: ${{ steps.compute-image-repo.outputs.RAFT_ANN_BENCH_IMAGE_REPO }}
      RAFT_ANN_BENCH_DATASETS_IMAGE_REPO: ${{ steps.compute-image-repo.outputs.RAFT_ANN_BENCH_DATASETS_IMAGE_REPO }}
      RAFT_ANN_BENCH_CPU_IMAGE_REPO: ${{ steps.compute-image-repo.outputs.RAFT_ANN_BENCH_CPU_IMAGE_REPO }}
      MATRIX: ${{ steps.compute-matrix.outputs.MATRIX }}
      TEST_MATRIX: ${{ steps.compute-test-matrix.outputs.TEST_MATRIX }}
      BASE_TAG_PREFIX: ${{ steps.compute-tag-prefix.outputs.BASE_TAG_PREFIX }}
      NOTEBOOKS_TAG_PREFIX: ${{ steps.compute-tag-prefix.outputs.NOTEBOOKS_TAG_PREFIX }}
      RAFT_ANN_BENCH_TAG_PREFIX: ${{ steps.compute-tag-prefix.outputs.RAFT_ANN_BENCH_TAG_PREFIX }}
      RAFT_ANN_BENCH_DATASETS_TAG_PREFIX: ${{ steps.compute-tag-prefix.outputs.RAFT_ANN_BENCH_DATASETS_TAG_PREFIX }}
      RAFT_ANN_BENCH_CPU_TAG_PREFIX: ${{ steps.compute-tag-prefix.outputs.RAFT_ANN_BENCH_CPU_TAG_PREFIX }}
      RAPIDS_VER: ${{ steps.compute-rapids-ver.outputs.RAPIDS_VER }}
      ALPHA_TAG: ${{ steps.compute-rapids-ver.outputs.ALPHA_TAG }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Compute matrix
        id: compute-matrix
        run: |
          MATRIX=$(ci/compute-matrix.sh)
          echo "MATRIX=${MATRIX}" | tee -a ${GITHUB_OUTPUT}
      - name: Compute tag prefix
        id: compute-tag-prefix
        run: |
          TAG_PREFIX=""
          if [ "${{ inputs.build_type }}" = "pull-request" ]; then
            pr_num="${GITHUB_REF_NAME##*/}"
            BASE_TAG_PREFIX="docker-${pr_num}-"
            NOTEBOOKS_TAG_PREFIX="docker-notebooks-${pr_num}-"
            RAFT_ANN_BENCH_TAG_PREFIX="docker-raft-ann-bench-${pr_num}-"
            RAFT_ANN_BENCH_DATASETS_TAG_PREFIX="docker-raft-ann-bench-datasets-${pr_num}-"
            RAFT_ANN_BENCH_CPU_TAG_PREFIX="docker-raft-ann-bench-cpu-${pr_num}-"
          fi
          echo "BASE_TAG_PREFIX=${BASE_TAG_PREFIX}" | tee -a ${GITHUB_OUTPUT}
          echo "NOTEBOOKS_TAG_PREFIX=${NOTEBOOKS_TAG_PREFIX}" | tee -a ${GITHUB_OUTPUT}
          echo "RAFT_ANN_BENCH_TAG_PREFIX=${RAFT_ANN_BENCH_TAG_PREFIX}" | tee -a ${GITHUB_OUTPUT}
          echo "RAFT_ANN_BENCH_DATASETS_TAG_PREFIX=${RAFT_ANN_BENCH_DATASETS_TAG_PREFIX}" | tee -a ${GITHUB_OUTPUT}
          echo "RAFT_ANN_BENCH_CPU_TAG_PREFIX=${RAFT_ANN_BENCH_CPU_TAG_PREFIX}" | tee -a ${GITHUB_OUTPUT}
      - name: Compute image repo
        id: compute-image-repo
        run: |
          base_repo="base"
          notebooks_repo="notebooks"
          raft_ann_bench_repo="raft-ann-bench"
          raft_ann_bench_datasets_repo="raft-ann-bench-datasets"
          raft_ann_bench_cpu_repo="raft-ann-bench-cpu"
          if [ "${{ inputs.build_type }}" = "pull-request" ]; then
            base_repo="staging"
            notebooks_repo="staging"
            raft_ann_bench_repo="staging"
            raft_ann_bench_datasets_repo="staging"
            raft_ann_bench_cpu_repo="staging"
          fi
          echo "BASE_IMAGE_REPO=${base_repo}" | tee -a ${GITHUB_OUTPUT}
          echo "NOTEBOOKS_IMAGE_REPO=${notebooks_repo}" | tee -a ${GITHUB_OUTPUT}
          echo "RAFT_ANN_BENCH_IMAGE_REPO=${raft_ann_bench_repo}" | tee -a ${GITHUB_OUTPUT}
          echo "RAFT_ANN_BENCH_DATASETS_IMAGE_REPO=${raft_ann_bench_datasets_repo}" | tee -a ${GITHUB_OUTPUT}
          echo "RAFT_ANN_BENCH_CPU_IMAGE_REPO=${raft_ann_bench_cpu_repo}" | tee -a ${GITHUB_OUTPUT}
      - name: Compute RAPIDS_VER
        id: compute-rapids-ver
        run: |
          GIT_DESCRIBE_TAG="$(git describe --tags --abbrev=0)"
          GIT_DESCRIBE_TAG="${GIT_DESCRIBE_TAG:1}" # remove leading 'v'
          ALPHA_TAG=""
          if [[ $GIT_DESCRIBE_TAG =~ [a-z] ]]; then
            rapids-logger "Most recent tag is an alpha tag"
            ALPHA_TAG="a"
          fi
          RAPIDS_VER="$(echo $GIT_DESCRIBE_TAG | awk 'BEGIN{FS=OFS="."} NF--')" # Convert full tag to YY.MM

          echo "RAPIDS_VER=${RAPIDS_VER}" | tee -a ${GITHUB_OUTPUT}
          echo "ALPHA_TAG=${ALPHA_TAG}" | tee -a ${GITHUB_OUTPUT}
      - name: Compute test matrix
        id: compute-test-matrix
        run: |
          TEST_MATRIX=$(yq '.${{ inputs.build_type }}' matrix-test.yaml)
          export TEST_MATRIX

          echo "TEST_MATRIX=$(yq -n -o json 'env(TEST_MATRIX)' | jq -c '{include: .}')" | tee --append "${GITHUB_OUTPUT}"
  build:
    needs: compute-matrix
    strategy:
      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
      fail-fast: false
    secrets: inherit
    uses: ./.github/workflows/build-image.yml
    with:
      ARCHES: ${{ toJSON(matrix.ARCHES) }}
      CUDA_VER: ${{ matrix.CUDA_VER }}
      LINUX_DISTRO: ${{ matrix.LINUX_DISTRO }}
      LINUX_DISTRO_VER: ${{ matrix.LINUX_DISTRO_VER }}
      LINUX_VER: ${{ matrix.LINUX_VER }}
      PYTHON_VER: ${{ matrix.PYTHON_VER }}
      RAPIDS_VER: ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
      DASK_SQL_VER: ${{ matrix.DASK_SQL_VER }}
      BASE_TAG:
        "rapidsai/${{ needs.compute-matrix.outputs.BASE_IMAGE_REPO }}:\
        ${{ needs.compute-matrix.outputs.BASE_TAG_PREFIX }}\
        ${{ needs.compute-matrix.outputs.RAPIDS_VER }}\
        ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-\
        cuda${{ matrix.CUDA_TAG }}-\
        py${{ matrix.PYTHON_VER }}"
      NOTEBOOKS_TAG:
        "rapidsai/${{ needs.compute-matrix.outputs.NOTEBOOKS_IMAGE_REPO }}:\
        ${{ needs.compute-matrix.outputs.NOTEBOOKS_TAG_PREFIX }}\
        ${{ needs.compute-matrix.outputs.RAPIDS_VER }}\
        ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-\
        cuda${{ matrix.CUDA_TAG }}-\
        py${{ matrix.PYTHON_VER }}"
      RAFT_ANN_BENCH_TAG:
        "rapidsai/${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_IMAGE_REPO }}:\
        ${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_TAG_PREFIX }}\
        ${{ needs.compute-matrix.outputs.RAPIDS_VER }}\
        ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-\
        cuda${{ matrix.CUDA_TAG }}-\
        py${{ matrix.PYTHON_VER }}"
      RAFT_ANN_BENCH_DATASETS_TAG:
        "rapidsai/${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_DATASETS_IMAGE_REPO }}:\
        ${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_DATASETS_TAG_PREFIX }}\
        ${{ needs.compute-matrix.outputs.RAPIDS_VER }}\
        ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-\
        cuda${{ matrix.CUDA_TAG }}-\
        py${{ matrix.PYTHON_VER }}"
      RAFT_ANN_BENCH_CPU_TAG:
        "rapidsai/${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_CPU_IMAGE_REPO }}:\
        ${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_CPU_TAG_PREFIX }}\
        ${{ needs.compute-matrix.outputs.RAPIDS_VER }}\
        ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-\
        py${{ matrix.PYTHON_VER }}"
  validate:
    needs: [build, compute-matrix]
    strategy:
      matrix: ${{ fromJSON(needs.compute-matrix.outputs.TEST_MATRIX) }}
      fail-fast: false
    secrets: inherit
    uses: ./.github/workflows/validate.yml
    with:
      ARCH: ${{ matrix.ARCH }}
      CONTAINER_CANARY_VERSION: 'v0.3.1'
      CUDA_VER: ${{ matrix.CUDA_VER }}
      GPU: ${{ matrix.GPU }}
      DRIVER: ${{ matrix.DRIVER }}
      PYTHON_VER: ${{ matrix.PYTHON_VER }}
      # images to test
      BASE_TAG:
        "rapidsai/${{ needs.compute-matrix.outputs.BASE_IMAGE_REPO }}:\
        ${{ needs.compute-matrix.outputs.BASE_TAG_PREFIX }}\
        ${{ needs.compute-matrix.outputs.RAPIDS_VER }}\
        ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-\
        cuda${{ matrix.CUDA_VER }}-\
        py${{ matrix.PYTHON_VER }}"-\
        ${{ matrix.ARCH }}"
      NOTEBOOKS_TAG:
        "rapidsai/${{ needs.compute-matrix.outputs.NOTEBOOKS_IMAGE_REPO }}:\
        ${{ needs.compute-matrix.outputs.NOTEBOOKS_TAG_PREFIX }}\
        ${{ needs.compute-matrix.outputs.RAPIDS_VER }}\
        ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-\
        cuda${{ matrix.CUDA_VER }}-\
        py${{ matrix.PYTHON_VER }}-\
        ${{ matrix.ARCH }}"
  test:
    needs: [build, compute-matrix, validate]
    if: inputs.run_tests
    strategy:
      matrix: ${{ fromJSON(needs.compute-matrix.outputs.TEST_MATRIX) }}
      fail-fast: false
    secrets: inherit
    uses: ./.github/workflows/test-notebooks.yml
    with:
      BUILD_TYPE: ${{ inputs.build_type }}
      ARCH: ${{ matrix.ARCH }}
      GPU: ${{ matrix.GPU }}
      DRIVER: ${{ matrix.DRIVER }}
      CUDA_VER: ${{ matrix.CUDA_VER }}
      PYTHON_VER: ${{ matrix.PYTHON_VER }}
      NOTEBOOKS_TAG:
        "rapidsai/${{ needs.compute-matrix.outputs.NOTEBOOKS_IMAGE_REPO }}:\
        ${{ needs.compute-matrix.outputs.NOTEBOOKS_TAG_PREFIX }}\
        ${{ needs.compute-matrix.outputs.RAPIDS_VER }}\
        ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-\
        cuda${{ matrix.CUDA_VER }}-\
        py${{ matrix.PYTHON_VER }}-\
        ${{ matrix.ARCH }}"
  build-multiarch-manifest:
    if: inputs.build_type == 'branch'
    needs: [build, compute-matrix, validate]
    strategy:
      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Login to DockerHub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }}
          password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}
      - name: Create multiarch manifest
        run: |
          base_source_tags=()
          notebooks_source_tags=()
          raft_ann_bench_source_tags=()
          raft_ann_bench_dataset_source_tags=()
          raft_ann_bench_cpu_source_tags=()

          base_tag_array=(
            rapidsai/${{ needs.compute-matrix.outputs.BASE_IMAGE_REPO }}:
            ${{ needs.compute-matrix.outputs.BASE_TAG_PREFIX }}
            ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
            ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-
            cuda${{ matrix.CUDA_TAG }}-
            py${{ matrix.PYTHON_VER }}
          )
          base_tag="$(printf %s "${base_tag_array[@]}" $'\n')" # Converts array to string w/o spaces

          notebooks_tag_array=(
            rapidsai/${{ needs.compute-matrix.outputs.NOTEBOOKS_IMAGE_REPO }}:
            ${{ needs.compute-matrix.outputs.NOTEBOOKS_TAG_PREFIX }}
            ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
            ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-
            cuda${{ matrix.CUDA_TAG }}-
            py${{ matrix.PYTHON_VER }}
          )
          notebooks_tag="$(printf %s "${notebooks_tag_array[@]}" $'\n')" # Converts array to string w/o spaces

          raft_ann_bench_tag_array=(
            rapidsai/${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_IMAGE_REPO }}:
            ${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_TAG_PREFIX }}
            ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
            ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-
            cuda${{ matrix.CUDA_TAG }}-
            py${{ matrix.PYTHON_VER }}
          )
          raft_ann_bench_tag="$(printf %s "${raft_ann_bench_tag_array[@]}" $'\n')" # Converts array to string w/o spaces

          raft_ann_bench_datasets_tag_array=(
            rapidsai/${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_DATASETS_IMAGE_REPO }}:
            ${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_DATASETS_TAG_PREFIX }}
            ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
            ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-
            cuda${{ matrix.CUDA_TAG }}-
            py${{ matrix.PYTHON_VER }}
          )
          raft_ann_bench_datasets_tag="$(printf %s "${raft_ann_bench_datasets_tag_array[@]}" $'\n')" # Converts array to string w/o spaces

          raft_ann_bench_cpu_tag_array=(
            rapidsai/${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_CPU_IMAGE_REPO }}:
            ${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_CPU_TAG_PREFIX }}
            ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
            ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-
            py${{ matrix.PYTHON_VER }}
          )
          raft_ann_bench_cpu_tag="$(printf %s "${raft_ann_bench_cpu_tag_array[@]}" $'\n')" # Converts array to string w/o spaces

          for arch in $(echo '${{ toJSON(matrix.ARCHES) }}' | jq .[] -r); do
            base_source_tags+=("${base_tag}-${arch}")
            notebooks_source_tags+=("${notebooks_tag}-${arch}")
            raft_ann_bench_source_tags+=("${raft_ann_bench_tag}-${arch}")
            raft_ann_bench_datasets_source_tags+=("${raft_ann_bench_datasets_tag}-${arch}")
            raft_ann_bench_cpu_source_tags+=("${raft_ann_bench_cpu_tag}-${arch}")
          done

          docker manifest create ${base_tag} ${base_source_tags[@]}
          docker manifest push ${base_tag}
          docker manifest create ${notebooks_tag} ${notebooks_source_tags[@]}
          docker manifest push ${notebooks_tag}
          docker manifest create ${raft_ann_bench_tag} ${raft_ann_bench_source_tags[@]}
          docker manifest push ${raft_ann_bench_tag}
          docker manifest create ${raft_ann_bench_datasets_tag} ${raft_ann_bench_datasets_source_tags[@]}
          docker manifest push ${raft_ann_bench_datasets_tag}
          docker manifest create ${raft_ann_bench_cpu_tag} ${raft_ann_bench_cpu_source_tags[@]}
          docker manifest push ${raft_ann_bench_cpu_tag}
  delete-temp-images:
    if: always()
    needs: [compute-matrix, build, test, build-multiarch-manifest]
    strategy:
      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
    runs-on: ubuntu-latest
    steps:
      - name: Remove temporary images
        run: |
          HUB_TOKEN=$(
            curl -s -H "Content-Type: application/json" \
              -X POST \
              -d "{\"username\": \"${{ secrets.GPUCIBOT_DOCKERHUB_USER }}\", \"password\": \"${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}\"}" \
              https://hub.docker.com/v2/users/login/ | jq -r .token \
          )
          echo "::add-mask::${HUB_TOKEN}"

          org="rapidsai"
          base_repo="${{ needs.compute-matrix.outputs.BASE_IMAGE_REPO }}"
          base_tag_array=(
            ${{ needs.compute-matrix.outputs.BASE_TAG_PREFIX }}
            ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
            ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-
            cuda${{ matrix.CUDA_TAG }}-
            py${{ matrix.PYTHON_VER }}
          )
          base_tag="$(printf %s "${base_tag_array[@]}" $'\n')" # Converts array to string w/o spaces


          notebooks_repo="${{ needs.compute-matrix.outputs.NOTEBOOKS_IMAGE_REPO }}"
          notebooks_tag_array=(
            ${{ needs.compute-matrix.outputs.NOTEBOOKS_TAG_PREFIX }}
            ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
            ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-
            cuda${{ matrix.CUDA_TAG }}-
            py${{ matrix.PYTHON_VER }}
          )
          notebooks_tag="$(printf %s "${notebooks_tag_array[@]}" $'\n')" # Converts array to string w/o spaces

          raft_ann_bench_repo="${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_IMAGE_REPO }}"
          raft_ann_bench_tag_array=(
            ${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_TAG_PREFIX }}
            ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
            ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-
            cuda${{ matrix.CUDA_TAG }}-
            py${{ matrix.PYTHON_VER }}
          )
          raft_ann_bench_tag="$(printf %s "${raft_ann_bench_tag_array[@]}" $'\n')" # Converts array to string w/o spaces

          raft_ann_bench_datasets_repo="${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_DATASETS_IMAGE_REPO }}"
          raft_ann_bench_datasets_tag_array=(
            ${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_DATASETS_TAG_PREFIX }}
            ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
            ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-
            cuda${{ matrix.CUDA_TAG }}-
            py${{ matrix.PYTHON_VER }}
          )
          raft_ann_bench_datasets_tag="$(printf %s "${raft_ann_bench_datasets_tag_array[@]}" $'\n')" # Converts array to string w/o spaces

          raft_ann_bench_cpu_repo="${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_CPU_IMAGE_REPO }}"
          raft_ann_bench_cpu_tag_array=(
            ${{ needs.compute-matrix.outputs.RAFT_ANN_BENCH_CPU_TAG_PREFIX }}
            ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
            ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-
            py${{ matrix.PYTHON_VER }}
          )
          raft_ann_bench_cpu_tag="$(printf %s "${raft_ann_bench_cpu_tag_array[@]}" $'\n')" # Converts array to string w/o spaces

          for arch in $(echo '${{ toJSON(matrix.ARCHES) }}' | jq .[] -r); do
            curl -i -X DELETE \
              -H "Accept: application/json" \
              -H "Authorization: JWT $HUB_TOKEN" \
              "https://hub.docker.com/v2/repositories/$org/$base_repo/tags/$base_tag-$arch/"
            curl -i -X DELETE \
              -H "Accept: application/json" \
              -H "Authorization: JWT $HUB_TOKEN" \
              "https://hub.docker.com/v2/repositories/$org/$notebooks_repo/tags/$notebooks_tag-$arch/"
            curl -i -X DELETE \
              -H "Accept: application/json" \
              -H "Authorization: JWT $HUB_TOKEN" \
              "https://hub.docker.com/v2/repositories/$org/$raft_ann_bench_repo/tags/$raft_ann_bench_tag-$arch/"
            curl -i -X DELETE \
              -H "Accept: application/json" \
              -H "Authorization: JWT $HUB_TOKEN" \
              "https://hub.docker.com/v2/repositories/$org/$raft_ann_bench_datasets_repo/tags/$raft_ann_bench_datasets_tag-$arch/"
            curl -i -X DELETE \
              -H "Accept: application/json" \
              -H "Authorization: JWT $HUB_TOKEN" \
              "https://hub.docker.com/v2/repositories/$org/$raft_ann_bench_cpu_repo/tags/$raft_ann_bench_cpu_tag-$arch/"
          done