Update Dockerfiles and CI (ros-navigation#1942)

* Use colcon to check for build failure
instead of canary file

* Simplify if case for shell

* Move extra dockerfiles to docker related folder

* Update Dockerfile location

* Stage refactor

* Use build kit

* Update underlay and overlay build

* Dockerfile tweek

* Rename underlay .repos file

* Remove uneeded key

* Clone matching distro branch by default

* Add optional directive to RUN colcon test

* Add comments

* Skip the use of symlinks

* Roll back to nightly image with all rmws
now that connext is fixed

* Add slam_toolbox to skipped rosdep keys

* Set DEBIAN_FRONTEND env to noninteractive

* Update to tag foxy

* Updated to latest codecov orb

.circleci/config.yml

@@ -1,6 +1,6 @@
 version: 2.1
 orbs:
-  codecov: codecov/codecov@1.0.5
+  codecov: codecov/codecov@1.1.1
 
 _commands:
   common_commands: &common_commands
@@ -103,16 +103,17 @@ _commands:
                   name: Build Workspace | << parameters.workspace >>
                   working_directory: << parameters.workspace >>
                   command: |
-                    if [ -d install ] && [ ! -f build_failed ]
+                    BUILD_UNFINISHED=$(colcon list --packages-skip-build-finished)
+                    BUILD_FAILED=$(colcon list --packages-select-build-failed)
+                    if [ -n "$BUILD_UNFINISHED" ] || [ -n "$BUILD_FAILED" ]
                     then
-                      echo "Skipping Build"
-                    else
                       . << parameters.underlay >>/install/setup.sh
                       rm -rf build install log
                       colcon build \
                         --symlink-install \
                         --mixin << parameters.mixins >>
-                      rm -f build_failed
+                    else
+                      echo "Skipping Build"
                     fi
               - save_to_cache:
                   key: << parameters.key >>
@@ -192,16 +193,16 @@ _steps:
       name: Post Checkout
       command: |
         if ! cmp \
-          $OVERLAY_WS/src/navigation2/tools/ros2_dependencies.repos \
-          $UNDERLAY_WS/ros2_dependencies.repos >/dev/null 2>&1
+          $OVERLAY_WS/src/navigation2/tools/underlay.repos \
+          $UNDERLAY_WS/underlay.repos >/dev/null 2>&1
         then
           echo "Cleaning Underlay"
           rm -rf $UNDERLAY_WS/*
-          cp $OVERLAY_WS/src/navigation2/tools/ros2_dependencies.repos \
-            $UNDERLAY_WS/ros2_dependencies.repos
+          cp $OVERLAY_WS/src/navigation2/tools/underlay.repos \
+            $UNDERLAY_WS/underlay.repos
           mkdir -p $UNDERLAY_WS/src
           vcs import $UNDERLAY_WS/src \
-            < $UNDERLAY_WS/ros2_dependencies.repos
+            < $UNDERLAY_WS/underlay.repos
         fi
   install_underlay_dependencies: &install_underlay_dependencies
     install_dependencies:
@@ -320,6 +321,7 @@ _environments:
     MAKEFLAGS: "-j 1 -l 2"
     RCUTILS_CONSOLE_STDOUT_LINE_BUFFERED: "0"
     RETEST_UNTIL_PASS: "2"
+    DEBIAN_FRONTEND: "noninteractive"
 
 executors:
   debug_exec:

.dockerhub/debug/hooks/build

@@ -12,4 +12,4 @@ docker build \
     --build-arg FAIL_ON_BUILD_FAILURE \
     --build-arg UNDERLAY_MIXINS \
     --build-arg OVERLAY_MIXINS \
-    --file ./Dockerfile ../../.
+    --file ../../Dockerfile ../../.

tools/release.Dockerfile → .dockerhub/distro.Dockerfile

@@ -6,14 +6,14 @@
 #
 # Example build command:
 # export DOCKER_BUILDKIT=1
-# export FROM_IMAGE="ros:eloquent"
+# export FROM_IMAGE="ros:foxy"
 # export OVERLAY_MIXINS="release ccache"
-# docker build -t nav2:release_branch \
+# docker build -t nav2:foxy \
 #   --build-arg FROM_IMAGE \
 #   --build-arg OVERLAY_MIXINS \
-#   -f release_branch.Dockerfile ./
+#   -f distro.Dockerfile ../
 
-ARG FROM_IMAGE=ros:eloquent
+ARG FROM_IMAGE=ros:foxy
 ARG OVERLAY_WS=/opt/overlay_ws
 
 # multi-stage for caching
@@ -43,6 +43,7 @@ RUN mkdir -p /tmp/opt && \
 
 # multi-stage for building
 FROM $FROM_IMAGE AS builder
+ARG DEBIAN_FRONTEND=noninteractive
 
 # edit apt for caching
 RUN cp /etc/apt/apt.conf.d/docker-clean /etc/apt/ && \
@@ -88,15 +89,13 @@ RUN sed --in-place \
       's|^source .*|source "$OVERLAY_WS/install/setup.bash"|' \
       /ros_entrypoint.sh
 
-# ARG RUN_TESTS
-# ARG FAIL_ON_TEST_FAILURE
-# RUN if [ -z "$RUN_TESTS" ]; then \
-#         colcon test \
-#           --mixin $OVERLAY_MIXINS \
-#           --ctest-args --test-regex "test_.*"; \
-#         if [ -z "$FAIL_ON_TEST_FAILURE" ]; then \
-#             colcon test-result; \
-#         else \
-#             colcon test-result || true; \
-#         fi \
-#     fi
+# test overlay build
+ARG RUN_TESTS
+ARG FAIL_ON_TEST_FAILURE=Ture
+RUN if [ -n "$RUN_TESTS" ]; then \
+        . $OVERLAY_WS/install/setup.sh && \
+        colcon test \
+          --mixin $OVERLAY_MIXINS \
+        && colcon test-result \
+          || ([ -z "$FAIL_ON_TEST_FAILURE" ] || exit 1) \
+    fi

.dockerhub/release/hooks/build

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -ex
 
-export FROM_IMAGE=osrf/ros2:nightly
+export FROM_IMAGE=osrf/ros2:nightly-rmw-nonfree
 export FAIL_ON_BUILD_FAILURE=""
 export UNDERLAY_MIXINS="release ccache"
 export OVERLAY_MIXINS="release ccache"
@@ -12,4 +12,4 @@ docker build \
     --build-arg FAIL_ON_BUILD_FAILURE \
     --build-arg UNDERLAY_MIXINS \
     --build-arg OVERLAY_MIXINS \
-    --file ./Dockerfile ../../.
+    --file ../../Dockerfile ../../.

.dockerhub/source.Dockerfile

@@ -0,0 +1,177 @@
+# syntax=docker/dockerfile:experimental
+
+# Use experimental buildkit for faster builds
+# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/experimental.md
+# Use `--progress=plain` to use plane stdout for docker build
+#
+# Example build command:
+# This determines which version of the ROS2 code base to pull
+# export ROS2_BRANCH=main
+# export DOCKER_BUILDKIT=1
+# docker build \
+#   --tag nav2:source \
+#   --file source.Dockerfile ../
+#
+# Omit the `--no-cache` if you know you don't need to break the cache.
+# We're only building on top of a ros2 devel image to get the basics
+# prerequisites installed such as the apt source, rosdep, etc. We don't want to
+# actually use any of the ros release packages. Instead we are going to build
+# everything from source in one big workspace.
+
+ARG FROM_IMAGE=osrf/ros2:devel
+ARG UNDERLAY_WS=/opt/underlay_ws
+ARG OVERLAY_WS=/opt/overlay_ws
+
+# multi-stage for caching
+FROM $FROM_IMAGE AS cacher
+
+# clone ros2 source
+ARG ROS2_BRANCH=master
+ARG ROS2_REPO=https://github.com/ros2/ros2.git
+WORKDIR $ROS2_WS/src
+RUN git clone $ROS2_REPO -b $ROS2_BRANCH && \
+    vcs import ./ < ros2/ros2.repos && \
+    find ./ -name ".git" | xargs rm -rf
+
+# clone underlay source
+ARG UNDERLAY_WS
+WORKDIR $UNDERLAY_WS/src
+COPY ./tools/underlay.repos ../
+RUN vcs import ./ < ../underlay.repos && \
+    find ./ -name ".git" | xargs rm -rf
+
+# copy overlay source
+ARG OVERLAY_WS
+WORKDIR $OVERLAY_WS/src
+COPY ./ ./ros-planning/navigation2
+RUN colcon list --names-only | cat > ../packages.txt
+
+# remove skiped packages
+WORKDIR /opt
+RUN find ./ \
+      -name "AMENT_IGNORE" -o \
+      -name "CATKIN_IGNORE" -o \
+      -name "COLCON_IGNORE" \
+      | xargs dirname | xargs rm -rf || true && \
+    colcon list --paths-only \
+      --packages-skip-up-to  \
+        $(cat $OVERLAY_WS/packages.txt | xargs) \
+      | xargs rm -rf
+
+# copy manifests for caching
+RUN mkdir -p /tmp/opt && \
+    find ./ -name "package.xml" | \
+      xargs cp --parents -t /tmp/opt
+
+# multi-stage for building
+FROM $FROM_IMAGE AS builder
+ARG DEBIAN_FRONTEND=noninteractive
+
+# edit apt for caching
+RUN cp /etc/apt/apt.conf.d/docker-clean /etc/apt/ && \
+    echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' \
+      > /etc/apt/apt.conf.d/docker-clean
+
+# install packages
+RUN --mount=type=cache,target=/var/cache/apt \
+    --mount=type=cache,target=/var/lib/apt \
+    apt-get update && apt-get install -q -y \
+      ccache \
+      libasio-dev \
+      libtinyxml2-dev \
+    && rosdep update
+
+ENV ROS_VERSION=2 \
+    ROS_PYTHON_VERSION=3
+
+# install ros2 dependencies
+WORKDIR $ROS2_WS
+COPY --from=cacher /tmp/$ROS2_WS ./
+COPY ./tools/skip_keys.txt /tmp/
+RUN --mount=type=cache,target=/var/cache/apt \
+    --mount=type=cache,target=/var/lib/apt \
+    apt-get update && rosdep install -q -y \
+      --from-paths src \
+      --ignore-src \
+      --skip-keys " \
+        $(cat /tmp/skip_keys.txt | xargs) \
+        "
+
+# build ros2 source
+COPY --from=cacher $ROS2_WS ./
+ARG ROS2_MIXINS="release ccache"
+RUN --mount=type=cache,target=/root/.ccache \
+    colcon build \
+      --symlink-install \
+      --mixin $ROS2_MIXINS
+
+# install underlay dependencies
+ARG UNDERLAY_WS
+WORKDIR $UNDERLAY_WS
+COPY --from=cacher /tmp/$UNDERLAY_WS ./
+RUN --mount=type=cache,target=/var/cache/apt \
+    --mount=type=cache,target=/var/lib/apt \
+    . $ROS2_WS/install/setup.sh && \
+    apt-get update && rosdep install -q -y \
+      --from-paths src \
+        $ROS2_WS/src \
+      --ignore-src \
+      --skip-keys " \
+        $(cat /tmp/skip_keys.txt | xargs) \
+      "
+
+# build underlay source
+COPY --from=cacher $UNDERLAY_WS ./
+ARG UNDERLAY_MIXINS="release ccache"
+RUN --mount=type=cache,target=/root/.ccache \
+    . $ROS2_WS/install/setup.sh && \
+    colcon build \
+      --symlink-install \
+      --mixin $UNDERLAY_MIXINS
+
+# install overlay dependencies
+ARG OVERLAY_WS
+WORKDIR $OVERLAY_WS
+COPY --from=cacher /tmp/$OVERLAY_WS ./
+RUN --mount=type=cache,target=/var/cache/apt \
+    --mount=type=cache,target=/var/lib/apt \
+    . $UNDERLAY_WS/install/setup.sh && \
+    apt-get update && rosdep install -q -y \
+      --from-paths src \
+        $ROS2_WS/src \
+        $UNDERLAY_WS/src \
+      --ignore-src \
+      --skip-keys " \
+        $(cat /tmp/skip_keys.txt | xargs) \
+      "
+
+# build overlay source
+COPY --from=cacher $OVERLAY_WS ./
+ARG OVERLAY_MIXINS="release ccache"
+RUN --mount=type=cache,target=/root/.ccache \
+    . $UNDERLAY_WS/install/setup.sh && \
+    colcon build \
+      --symlink-install \
+      --mixin $OVERLAY_MIXINS
+
+# restore apt for docker
+RUN mv /etc/apt/docker-clean /etc/apt/apt.conf.d/ && \
+    rm -rf /var/lib/apt/lists/
+
+# source overlay from entrypoint
+ENV UNDERLAY_WS $UNDERLAY_WS
+ENV OVERLAY_WS $OVERLAY_WS
+RUN sed --in-place \
+      's|^source .*|source "$OVERLAY_WS/install/setup.bash"|' \
+      /ros_entrypoint.sh
+
+# test overlay build
+ARG RUN_TESTS
+ARG FAIL_ON_TEST_FAILURE=Ture
+RUN if [ -n "$RUN_TESTS" ]; then \
+        . $OVERLAY_WS/install/setup.sh && \
+        colcon test \
+          --mixin $OVERLAY_MIXINS \
+        && colcon test-result \
+          || ([ -z "$FAIL_ON_TEST_FAILURE" ] || exit 1) \
+    fi

Dockerfile

@@ -16,8 +16,8 @@ FROM $FROM_IMAGE AS cacher
 # clone underlay source
 ARG UNDERLAY_WS
 WORKDIR $UNDERLAY_WS/src
-COPY ./tools/ros2_dependencies.repos ../
-RUN vcs import ./ < ../ros2_dependencies.repos && \
+COPY ./tools/underlay.repos ../
+RUN vcs import ./ < ../underlay.repos && \
     find ./ -name ".git" | xargs rm -rf
 
 # copy overlay source
@@ -35,6 +35,7 @@ RUN mkdir -p /tmp/opt && \
 
 # multi-stage for building
 FROM $FROM_IMAGE AS builder
+ARG DEBIAN_FRONTEND=noninteractive
 
 # install CI dependencies
 RUN apt-get update && apt-get install -q -y \
@@ -65,10 +66,7 @@ RUN . /opt/ros/$ROS_DISTRO/setup.sh && \
       --symlink-install \
       --mixin $UNDERLAY_MIXINS \
       --event-handlers console_direct+ \
-    || touch build_failed && \
-    if [ -f build_failed ] && [ -n "$FAIL_ON_BUILD_FAILURE" ]; then \
-      exit 1; \
-    fi
+    || ([ -z "$FAIL_ON_BUILD_FAILURE" ] || exit 1)
 
 # install overlay dependencies
 ARG OVERLAY_WS
@@ -91,14 +89,22 @@ RUN . $UNDERLAY_WS/install/setup.sh && \
     colcon build \
       --symlink-install \
       --mixin $OVERLAY_MIXINS \
-    || touch build_failed && \
-    if [ -f build_failed ] && [ -n "$FAIL_ON_BUILD_FAILURE" ]; then \
-      exit 1; \
-    fi
+    || ([ -z "$FAIL_ON_BUILD_FAILURE" ] || exit 1)
 
 # source overlay from entrypoint
 ENV UNDERLAY_WS $UNDERLAY_WS
 ENV OVERLAY_WS $OVERLAY_WS
 RUN sed --in-place \
       's|^source .*|source "$OVERLAY_WS/install/setup.bash"|' \
       /ros_entrypoint.sh
+
+# test overlay build
+ARG RUN_TESTS
+ARG FAIL_ON_TEST_FAILURE=Ture
+RUN if [ -n "$RUN_TESTS" ]; then \
+        . $OVERLAY_WS/install/setup.sh && \
+        colcon test \
+          --mixin $OVERLAY_MIXINS \
+        && colcon test-result \
+          || ([ -z "$FAIL_ON_TEST_FAILURE" ] || exit 1) \
+    fi

doc/process/PreReleaseChecklist.md

@@ -31,7 +31,7 @@ to compare against ROS2 main.
 
 We want to ensure the correct version of all our dependencies have been released
 to the branch we are targeting. To do that, we skip the
-`ros2_dependencies.repos` install step and rely solely on rosdep to install
+`underlay.repos` install step and rely solely on rosdep to install
 everything.
 
 There is a dockerfile to do that as well, so run

tools/initial_ros_setup.sh

@@ -65,7 +65,7 @@ download_ros2_dependencies() {
   echo "Downloading the dependencies workspace"
   mkdir -p ros2_nav_dependencies_ws/src
   cd ros2_nav_dependencies_ws
-  vcs import src < ${CWD}/navigation2_ws/src/navigation2/tools/ros2_dependencies.repos
+  vcs import src < ${CWD}/navigation2_ws/src/navigation2/tools/underlay.repos
   return_to_root_dir
 }