From c120474c8f275ade84811018c6a853b9f34df519 Mon Sep 17 00:00:00 2001 From: Kenji Miyake <31987104+kenji-miyake@users.noreply.github.com> Date: Wed, 2 Feb 2022 18:41:31 +0900 Subject: [PATCH] feat: add basic Docker support (#14) * feat: add basic Docker support Signed-off-by: Kenji Miyake * ci: free disk space Signed-off-by: Kenji Miyake * ci: install sudo if not installed Signed-off-by: Kenji Miyake * chore: ignore errors for nektos/act Signed-off-by: Kenji Miyake * fix: add setup-qemu-action Signed-off-by: Kenji Miyake * ci: use self-hosted runner Signed-off-by: Kenji Miyake * ci: change permission of workspace for self-hosted runner Signed-off-by: Kenji Miyake * chore: show disk space after building docker images Signed-off-by: Kenji Miyake * fix: add --load to build.sh Signed-off-by: Kenji Miyake * feat: change WORKDIR for devel image Signed-off-by: Kenji Miyake * docs: update usage Signed-off-by: Kenji Miyake * docs: update docker/README.md Co-authored-by: Lalith Vipulananthan <63835446+LalithVipulananthan@users.noreply.github.com> Signed-off-by: Kenji Miyake * fix(Dockerfile): use /etc/bash.bashrc for rocker Signed-off-by: Kenji Miyake * docs: update docker/README.md Signed-off-by: Kenji Miyake * style: fix English Signed-off-by: Kenji Miyake * docs: apply review Signed-off-by: Kenji Miyake * docs: apply review Signed-off-by: Kenji Miyake * docs: apply review Signed-off-by: Kenji Miyake * ci: split ARM workflow Signed-off-by: Kenji Miyake * docs: apply review Signed-off-by: Kenji Miyake * docs: apply review Signed-off-by: Kenji Miyake Co-authored-by: Lalith Vipulananthan <63835446+LalithVipulananthan@users.noreply.github.com> --- .../actions/docker-build-and-push/action.yaml | 114 ++++++++++++++ .github/actions/free-disk-space/action.yaml | 36 +++++ .../workflows/docker-build-and-push-arm.yaml | 35 +++++ .github/workflows/docker-build-and-push.yaml | 30 ++++ .github/workflows/update-docker-manifest.yaml | 51 ++++++ .pre-commit-config.yaml | 5 + README.md | 4 +- docker/README.md | 146 ++++++++++++++++++ docker/autoware-universe/Dockerfile | 66 ++++++++ docker/autoware-universe/docker-bake.hcl | 19 +++ docker/build.sh | 15 ++ 11 files changed, 519 insertions(+), 2 deletions(-) create mode 100644 .github/actions/docker-build-and-push/action.yaml create mode 100644 .github/actions/free-disk-space/action.yaml create mode 100644 .github/workflows/docker-build-and-push-arm.yaml create mode 100644 .github/workflows/docker-build-and-push.yaml create mode 100644 .github/workflows/update-docker-manifest.yaml create mode 100644 docker/README.md create mode 100644 docker/autoware-universe/Dockerfile create mode 100644 docker/autoware-universe/docker-bake.hcl create mode 100755 docker/build.sh diff --git a/.github/actions/docker-build-and-push/action.yaml b/.github/actions/docker-build-and-push/action.yaml new file mode 100644 index 0000000000000..afef54ddd2af1 --- /dev/null +++ b/.github/actions/docker-build-and-push/action.yaml @@ -0,0 +1,114 @@ +name: docker-build-and-push +description: "" + +inputs: + bake-target: + description: "" + required: true + platforms: + description: "" + required: true + tag-suffix: + description: "" + required: true + +runs: + using: composite + steps: + - name: Check branch of workflow_dispatch + if: ${{ github.event_name == 'workflow_dispatch' && github.ref != 'refs/heads/main' }} + run: | + echo "workflow_dispatch is allowed only with the branch 'refs/heads/main', '${{ github.ref }}' is not allowed." + exit 1 + shell: bash + + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Install jq + run: | + sudo apt-get -y update + sudo apt-get -y install jq + shell: bash + + # workflow_dispatch: date + # scheduled: latest, date + # tag: semver + - name: Set Docker tags + id: set-docker-tags + run: | + tags=() + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + tags+=("{{date 'YYYYMMDD'}}") + else + tags+=("type=schedule,pattern=latest") + tags+=("type=schedule,pattern={{date 'YYYYMMDD'}}") + tags+=("type=semver,pattern={{version}}") + fi + + # Workaround for multiline strings + # https://github.community/t/set-output-truncates-multiline-strings/16852 + tags_multiline=$(printf "%s\n" "${tags[@]}") + tags_multiline="${tags_multiline//'%'/'%25'}" + tags_multiline="${tags_multiline//$'\n'/'%0A'}" + tags_multiline="${tags_multiline//$'\r'/'%0D'}" + + echo ::set-output name=tags::$tags_multiline + shell: bash + + - name: Docker meta for devel + id: meta-devel + uses: docker/metadata-action@v3 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-devel + flavor: | + latest=false + suffix=${{ inputs.tag-suffix }} + + - name: Docker meta for prebuilt + id: meta-prebuilt + uses: docker/metadata-action@v3 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }} + tags: ${{ steps.set-docker-tags.outputs.tags }} + bake-target: docker-metadata-action-prebuilt + flavor: | + latest=false + suffix=-prebuilt${{ inputs.tag-suffix }} + + - name: Login to GitHub Container Registry + if: ${{ github.event_name != 'pull_request' }} + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ github.token }} + + # For https://github.com/docker/buildx/issues/756 + - name: Merge json files + run: | + jq -s ".[0] * .[1]" \ + "${{ steps.meta-devel.outputs.bake-file }}" \ + "${{ steps.meta-prebuilt.outputs.bake-file }}" \ + > bake.json + shell: bash + + - name: Build and push + uses: docker/bake-action@v1 + with: + push: ${{ github.event_name != 'pull_request' }} + load: true + files: | + docker/${{ inputs.bake-target }}/docker-bake.hcl + bake.json + set: | + *.platform=${{ inputs.platforms }} + + - name: Output published tags + if: ${{ github.event_name != 'pull_request' }} + id: output-published-tags + run: | + echo ::set-output name=published-tags::$(jq ".target[].tags[]" < bake.json) + shell: bash diff --git a/.github/actions/free-disk-space/action.yaml b/.github/actions/free-disk-space/action.yaml new file mode 100644 index 0000000000000..04b0e85235ddb --- /dev/null +++ b/.github/actions/free-disk-space/action.yaml @@ -0,0 +1,36 @@ +name: free-disk-space +description: "" + +runs: + using: composite + steps: + - name: Install sudo + run: | + if ! (command -v sudo >/dev/null 2>&1); then + apt-get -y update + apt-get -y install sudo + fi + shell: bash + + # https://github.community/t/bug-strange-no-space-left-on-device-ioexceptions-on-github-runners/17616 + - name: Free disk space + run: | + df -h + + sudo apt-get -y purge \ + dotnet* \ + ghc* \ + php* \ + || true + sudo apt-get -y autoremove + sudo apt-get -y autoclean + + sudo rm -rf \ + /usr/local/lib/android \ + /usr/share/dotnet/ \ + /opt/ghc + + docker rmi $(docker image ls -aq) || true + + df -h + shell: bash diff --git a/.github/workflows/docker-build-and-push-arm.yaml b/.github/workflows/docker-build-and-push-arm.yaml new file mode 100644 index 0000000000000..d925b0add3e73 --- /dev/null +++ b/.github/workflows/docker-build-and-push-arm.yaml @@ -0,0 +1,35 @@ +name: docker-build-and-push-arm + +on: + push: + tags: + - v* + schedule: + - cron: 0 19 1,15 * * # run at 4 AM JST every two weeks + workflow_dispatch: + +jobs: + docker-build-and-push-arm: + runs-on: [self-hosted, linux, ARM64] + steps: + # https://github.com/actions/checkout/issues/211 + - name: Change permission of workspace + run: | + sudo chown -R $USER:$USER ${{ github.workspace }} + + - name: Check out repository + uses: actions/checkout@v2 + + - name: Free disk space + uses: ./.github/actions/free-disk-space + + - name: Build 'autoware-universe' + uses: ./.github/actions/docker-build-and-push + with: + bake-target: autoware-universe + platforms: linux/arm64 + tag-suffix: -arm64 + + - name: Show disk space + run: | + df -h diff --git a/.github/workflows/docker-build-and-push.yaml b/.github/workflows/docker-build-and-push.yaml new file mode 100644 index 0000000000000..d6bf5f9c5703d --- /dev/null +++ b/.github/workflows/docker-build-and-push.yaml @@ -0,0 +1,30 @@ +name: docker-build-and-push + +on: + push: + tags: + - v* + schedule: + - cron: 0 19 1,15 * * # run at 4 AM JST every two weeks + workflow_dispatch: + +jobs: + docker-build-and-push: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v2 + + - name: Free disk space + uses: ./.github/actions/free-disk-space + + - name: Build 'autoware-universe' + uses: ./.github/actions/docker-build-and-push + with: + bake-target: autoware-universe + platforms: linux/amd64 + tag-suffix: -amd64 + + - name: Show disk space + run: | + df -h diff --git a/.github/workflows/update-docker-manifest.yaml b/.github/workflows/update-docker-manifest.yaml new file mode 100644 index 0000000000000..5a27093a474ad --- /dev/null +++ b/.github/workflows/update-docker-manifest.yaml @@ -0,0 +1,51 @@ +name: update-docker-manifest + +on: + schedule: + - cron: 0 19 * * * # run at 4 AM JST + workflow_dispatch: + +jobs: + update-docker-manifest: + runs-on: ubuntu-latest + env: + PACKAGE_NAME: autoware-universe + steps: + - name: Login to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ github.token }} + + - name: Create Docker manifest + run: | + package_full_name=ghcr.io/${{ github.repository_owner }}/${{ env.PACKAGE_NAME }} + + url="https://api.github.com/orgs/${{ github.repository_owner }}/packages/container/${{ matrix.package-name }}/versions" + + echo "url: $url" + tags=$(curl -fsSL "$url" -H "Authorization: token ${{ github.token }}" | jq ".[].metadata.container.tags[]" | cut -d '"' -f 2) + + amd64_tags=$(echo "$tags" | grep "\-amd64" | sed "s/-amd64$//g") + arm64_tags=$(echo "$tags" | grep "\-arm64" | sed "s/-arm64$//g") + base_tags=$(printf "%s\n" "$amd64_tags" "$arm64_tags" | sort | uniq) + + echo "amd64_tags: "$amd64_tags"" + echo "arm64_tags: "$arm64_tags"" + echo "base_tags: "$base_tags"" + + for base_tag in $base_tags; do + amd64_tag="$package_full_name":$(echo "$tags" | grep "$base_tag\-amd64") + arm64_tag="$package_full_name":$(echo "$tags" | grep "$base_tag\-arm64") + + echo "base_tag: $base_tag" + echo "amd64_tag: $amd64_tag" + echo "arm64_tag: $arm64_tag" + + docker manifest create $package_full_name:$base_tag \ + $amd64_tag \ + $arm64_tag + + docker manifest push $package_full_name:$base_tag + done diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1bd167d3cb33e..8b4763b3981f5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -51,4 +51,9 @@ repos: - id: shfmt args: [-w, -s, -i=4] + - repo: https://github.com/AleksaC/hadolint-py + rev: v2.8.0 + hooks: + - id: hadolint + exclude: .svg diff --git a/README.md b/README.md index 857b0c337c1d6..4563c750d3927 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ This is the list of the prototype repositories and their roles. - This is a documentation repository for Autoware users and developers. - Since Autoware Core/Universe has multiple repositories, preparing a central documentation repository is more user-friendly than writing distributed documentation in each repository. -If you have any questions or ideas, please feel free to start a discussion on [GitHub Discussions in autowarefoundation/autoware](https://github.com/autowarefoundation/autoware/discussions). +If you have any questions or ideas, feel free to start a discussion on [GitHub Discussions in autowarefoundation/autoware](https://github.com/autowarefoundation/autoware/discussions). --- @@ -39,7 +39,7 @@ If you have any questions or ideas, please feel free to start a discussion on [G This script will install the development environment for Autoware, which cannot be installed by `rosdep`. -> Note: Please confirm the licenses before installing NVIDIA libraries. +> Note: Before installing NVIDIA libraries, confirm and agree with the licenses. - [CUDA](https://docs.nvidia.com/cuda/eula/index.html) - [cuDNN](https://docs.nvidia.com/deeplearning/cudnn/sla/index.html) diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000000000..c99a3011ffa47 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,146 @@ +# Docker images for Autoware + +We have two types of Docker image: `development` and `prebuilt`. + +1. The `development` image enables you to develop Autoware without setting up the local development environment. +2. The `prebuilt` image contains executables and enables you to try out Autoware quickly. + - Note that the prebuilt image is not designed for deployment on a real vehicle! + +**Note**: Before pulling these images, confirm and agree with the [NVIDIA Deep Learning Container license](https://developer.nvidia.com/ngc/nvidia-deep-learning-container-license). + +## Prerequisites + +- [Docker](https://docs.docker.com/engine/install/ubuntu/) +- [rocker](https://github.com/osrf/rocker) + - We use `rocker` to enable GUI applications such as `rviz` and `rqt` on Docker Containers. + - Refer to [here](http://wiki.ros.org/docker/Tutorials/GUI) for more details. + +The [setup script](../setup-dev-env.sh) will install these dependencies through the [docker role](../ansible/roles/docker/README.md). + +## Usage + +### Development image + +```bash +docker run --rm -it \ + -v {path_to_your_workspace}:/autoware \ + ghcr.io/autowarefoundation/autoware-universe:latest +``` + +To run with `rocker`: + +```bash +rocker --nvidia --x11 --user \ + --volume {path_to_your_workspace} \ + -- ghcr.io/autowarefoundation/autoware-universe:latest +``` + +If you locate your workspace under your home directory, you can use the `--home` option instead: + +```bash +rocker --nvidia --x11 --user --home \ + -- ghcr.io/autowarefoundation/autoware-universe:latest +``` + +To use a customized `.bashrc` for the container: + +```bash +rocker --nvidia --x11 --user --home \ + --volume $HOME/.bashrc.container:$HOME/.bashrc \ + -- ghcr.io/autowarefoundation/autoware-universe:latest +``` + +### Prebuilt image + +```bash +docker run --rm -it \ + ghcr.io/autowarefoundation/autoware-universe:latest-prebuilt +``` + +To run with `rocker`: + +```bash +rocker --nvidia --x11 --user \ + --volume {path_to_your_workspace} \ + -- ghcr.io/autowarefoundation/autoware-universe:latest-prebuilt +``` + +If you intend to use pre-existing data such as maps or Rosbags, modify the `--volume` options shown below. + +```bash +rocker --nvidia --x11 --user \ + --volume {path_to_your_workspace} \ + --volume {path_to_your_map_data} \ + --volume {path_to_your_log_data} \ + -- ghcr.io/autowarefoundation/autoware-universe:latest-prebuilt +``` + +## Building Docker images on your local machine + +If you want to build these images locally for development purposes, run the following command: + +```bash +cd autoware/ +./docker/build.sh +``` + +## Tips + +### Precautions for not using `rocker` + +If either image is run without `rocker`, then `root` privileges will be used. +This can affect your local environment as below: + +```sh-session +$ docker run --rm -it -v {path_to_your_workspace}:/autoware ghcr.io/autowarefoundation/autoware-universe:latest +# colcon build +# exit +$ rm build/COLCON_IGNORE +rm: remove write-protected regular empty file 'build/COLCON_IGNORE'? y +rm: cannot remove 'build/COLCON_IGNORE': Permission denied +``` + +To prevent this error occurring when rocker is not used, there are two suggested methods: + +1. Prepare a dedicated workspace for the docker image. +2. Use Visual Studio Code's [Remote - Containers](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) extension. + + To use the extension, the following settings can be used to create a user account in a similar way to `rocker. + Refer to [this document](https://code.visualstudio.com/remote/advancedcontainers/add-nonroot-user) for more details. + + ```jsonc + // .devcontainer/devcontainer.json + { + "name": "Autoware", + "build": { + "dockerfile": "Dockerfile" + }, + "remoteUser": "autoware", + "settings": { + "terminal.integrated.defaultProfile.linux": "bash" + } + } + ``` + + ```docker + # .devcontainer/Dockerfile + FROM ghcr.io/autowarefoundation/autoware-universe:latest + + ARG USERNAME=autoware + ARG USER_UID=1000 + ARG USER_GID=$USER_UID + + RUN groupadd --gid $USER_GID $USERNAME \ + && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME \ + && apt-get update \ + && apt-get install -y sudo \ + && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \ + && chmod 0440 /etc/sudoers.d/$USERNAME + ``` + +### Using Docker images other than `latest` + +There are also images versioned based on the `date` or `release tag`. +Use them when you need a fixed version of the image. + +The list of versions can be found [here](https://github.com/autowarefoundation/autoware/packages). diff --git a/docker/autoware-universe/Dockerfile b/docker/autoware-universe/Dockerfile new file mode 100644 index 0000000000000..8da0caaa94158 --- /dev/null +++ b/docker/autoware-universe/Dockerfile @@ -0,0 +1,66 @@ +FROM nvidia/cuda:11.4.1-devel-ubuntu20.04 as devel +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +## Install apt packages +# hadolint ignore=DL3008 +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends \ + git \ + ssh \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +## Copy files +COPY autoware.repos setup-dev-env.sh ansible-galaxy-requirements.yaml /autoware/ +COPY ansible/ /autoware/ansible/ +WORKDIR /autoware +RUN ls /autoware + +## Add GitHub to known hosts for private repositories +RUN mkdir -p ~/.ssh \ + && ssh-keyscan github.com >> ~/.ssh/known_hosts + +## Set up development environment +RUN --mount=type=ssh \ + ./setup-dev-env.sh -y --no-nvidia universe \ + && mkdir src \ + && vcs import src < autoware.repos \ + && rosdep update \ + && DEBIAN_FRONTEND=noninteractive rosdep install -y --ignore-src --from-paths src --rosdistro galactic \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +## Install cuDNN and TensorRT +ARG CUDNN_VERSION=8.2.2.26-1+cuda11.4 +ARG TENSORRT_VERSION=8.2.2-1+cuda11.4 +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends \ + libcudnn8=$CUDNN_VERSION \ + libcudnn8-dev=$CUDNN_VERSION \ + libnvinfer8=$TENSORRT_VERSION \ + libnvinfer-dev=$TENSORRT_VERSION \ + libnvinfer-plugin8=$TENSORRT_VERSION \ + libnvinfer-plugin-dev=$TENSORRT_VERSION \ + libnvonnxparsers8=$TENSORRT_VERSION \ + libnvonnxparsers-dev=$TENSORRT_VERSION \ + libnvparsers8=$TENSORRT_VERSION \ + libnvparsers-dev=$TENSORRT_VERSION \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +## Create entrypoint +# hadolint ignore=SC1091 +RUN echo "source /opt/ros/galactic/setup.bash" > /etc/bash.bashrc +CMD ["/bin/bash"] + +FROM devel as prebuilt +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +## Build and change permission for runtime data conversion +# hadolint ignore=SC1091 +RUN source /opt/ros/galactic/setup.bash \ + && colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release \ + && find /autoware/install -type d -exec chmod 777 {} \; + +## Create entrypoint +# hadolint ignore=SC1091 +RUN echo "source /autoware/install/setup.bash" > /etc/bash.bashrc +CMD ["/bin/bash"] diff --git a/docker/autoware-universe/docker-bake.hcl b/docker/autoware-universe/docker-bake.hcl new file mode 100644 index 0000000000000..fa5b3dff25f84 --- /dev/null +++ b/docker/autoware-universe/docker-bake.hcl @@ -0,0 +1,19 @@ +group "default" { + targets = ["devel", "prebuilt"] +} + +// For docker/metadata-action +target "docker-metadata-action-devel" {} +target "docker-metadata-action-prebuilt" {} + +target "devel" { + inherits = ["docker-metadata-action-devel"] + dockerfile = "docker/autoware-universe/Dockerfile" + target = "devel" +} + +target "prebuilt" { + inherits = ["docker-metadata-action-prebuilt"] + dockerfile = "docker/autoware-universe/Dockerfile" + target = "prebuilt" +} diff --git a/docker/build.sh b/docker/build.sh new file mode 100755 index 0000000000000..7149104c0953c --- /dev/null +++ b/docker/build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -e + +SCRIPT_DIR=$(readlink -f "$(dirname "$0")") +WORKSPACE_ROOT="$SCRIPT_DIR/../" + +# https://github.com/docker/buildx/issues/484 +export BUILDKIT_STEP_LOG_MAX_SIZE=10000000 + +docker buildx bake --load --progress=plain -f "$SCRIPT_DIR/autoware-universe/docker-bake.hcl" \ + --set "*.context=$WORKSPACE_ROOT" \ + --set "*.ssh=default" \ + --set "devel.tags=ghcr.io/autowarefoundation/autoware-universe:latest" \ + --set "prebuilt.tags=ghcr.io/autowarefoundation/autoware-universe:latest-prebuilt"