[DAPHNE-#491] CUDA Docker Container

* Scripts and config files for CUDA containers in Docker
* Other improvements to the docker build and run scripts
* Some more documentation

Closes #491

containers/Readme.md

@@ -34,8 +34,8 @@ in [GettingStarted.md](../doc/GettingStarted.md)
 The content of the scripts is quite self-explanatory but here's a shortlisting of what can be done:
 * **build-containers.sh**: Use this script to build your local Docker images. Alternatively, they can also be pulled 
 from Docker Hub: ``docker pull daphneeu/daphne-dev-interactive``
-* **run-daphne-dev-interactive.sh**: This script properly starts the daphne-dev-interactive container
-* **run-daphne-dev.sh**: Starts the non-interactive container, sets UID for permissions and mounts the current directory.
+* **run-docker-example.sh**: Use this as the starting point to customize scripts to launch the DAPHNE docker images from 
+your command line.
 
 ### Building a container
 To build the DAPHNE containers, use the provided ``build-containers.sh`` script contained in this directory.
@@ -48,7 +48,5 @@ with build.sh (which by defaults tries to build the dependencies in the ``thirdp
 ``./build.sh --no-deps --installPrefix /usr/local``
 
 ### TODO
-* **daphneeu/daphne** container with a prebuilt DAPHNE executable
-* CUDA and OneAPI images
 * Rebuilding the containers automatically for latest changes
 * Images of released versions of DAPHNE 

containers/build-containers.sh

@@ -14,9 +14,96 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+echo "This script is primarily meant for image maintainers. Users are advised to pull from https://hub.docker.com/u/daphneeu"
+echo
+echo "Comment/uncomment the docker command for the image(s) to be built"
+echo "If images are wrongly rebuilt from cache, add --no-cache to the docker command"
+echo "For verbose output add --progress=plain to the docker build command"
+
+GIT_REPO=daphne
+GIT_BRANCH="docker-cuda"
+GH_USER="corepointer"
+DAPHNE_REPO_URL="https://github.com/$GH_USER/$GIT_REPO.git"
+
 export DOCKER_BUILDKIT=1
+export PROGRESS_NO_TRUNC=1
+
+TIMESTAMP_DATE=$(date -I)
+TIMESTAMP_MINUTES=$(date +"%Y-%m-%dT%H%M") # use this to add hours and minutes to the timestamp
+TIMESTAMP_RFC3339=$(date --rfc-3339=seconds)
+BUILD_OUTPUT_LOGFILE=docker-build-log-$TIMESTAMP_MINUTES.txt
+
+function build_daphne() {
+    docker build --progress=plain --tag "$IMAGE_REPO:$DAPHNE_TAG" --tag "$IMAGE_REPO:latest" \
+        --build-arg NUM_CORES="$(nproc)" --build-arg TIMESTAMP="$TIMESTAMP_DATE" \
+        --build-arg GIT_HASH="$(curl -s https://api.github.com/repos/$GH_USER/$GIT_REPO/branches/$GIT_BRANCH | \
+                jq --raw-output '.commit["sha"]' -)" \
+        --build-arg BASE_IMAGE="$BASE_IMAGE" --build-arg FINAL_BASE_IMAGE="$FINAL_BASE_IMAGE"  \
+        --build-arg DAPHNE_REPO=$DAPHNE_REPO_URL --build-arg DAPHNE_BRANCH=$GIT_BRANCH \
+        --build-arg CREATION_DATE="$TIMESTAMP_RFC3339" --build-arg DAPHNE_BUILD_FLAGS="$DAPHNE_BUILD_FLAGS" \
+        --target "$DAPHNE_TARGET" -f ./daphne"$1".Dockerfile . 2>&1 | tee "$BUILD_OUTPUT_LOGFILE"
+}
+
+#------------------------------------------------------------------------------
+# Images for continuous integration (GitHub Actions)
+#------------------------------------------------------------------------------
+#BASE_IMAGE=ubuntu:20.04
+#FINAL_BASE_IMAGE=ubuntu:20.04
+#DAPHNE_TARGET=github-action
+#DAPHNE_TAG=$TIMESTAMP_DATE
+#IMAGE_REPO=daphneeu/$DAPHNE_TARGET
+#build_daphne -dev
+
+#------------------------------------------------------------------------------
+# Images for DAPHNE development
+#------------------------------------------------------------------------------
+#BASE_IMAGE=ubuntu:20.04
+#DAPHNE_TARGET=daphne-dev
+#DAPHNE_TAG=$TIMESTAMP_DATE
+#IMAGE_REPO=daphneeu/$DAPHNE_TARGET
+#build_daphne -dev
+
+#------------------------------------------------------------------------------
+# Images for running DAPHNE
+#------------------------------------------------------------------------------
+#BASE_IMAGE=daphneeu/daphne-deps-compile
+#FINAL_BASE_IMAGE=ubuntu:20.04
+#DAPHNE_TARGET=daphne
+#DAPHNE_TAG=ubuntu20.04_$TIMESTAMP_DATE
+#IMAGE_REPO=daphneeu/$DAPHNE_TARGET
+#DAPHNE_BUILD_FLAGS="--mpi"
+#build_daphne
+
+#------------------------------------------------------------------------------
+# Images for DAPHNE development (CUDA)
+#------------------------------------------------------------------------------
+#DAPHNE_TARGET=daphne-dev
+#CUDA_TAG=12.0.1-cudnn8-devel-ubuntu20.04
+#DAPHNE_TAG=${CUDA_TAG}_$TIMESTAMP_DATE
+#IMAGE_REPO=daphneeu/$DAPHNE_TARGET-cuda
+#FINAL_BASE_IMAGE=nvidia/cuda:$CUDA_TAG
+#BASE_IMAGE=ubuntu:20.04
+#build_daphne -dev
 
-docker build -t daphneeu/daphne-dev --build-arg NUM_CORES="$(nproc)" \
-    --build-arg DAPHNE_REPO="https://github.com/daphne-eu/daphne.git" --build-arg DAPHNE_BRANCH="main" -f ./daphne-dev.Dockerfile .
+#-----------------------------------------------------------------------------
+# Images for running DAPHNE (CUDA)
+#------------------------------------------------------------------------------
+#DAPHNE_TARGET=daphne
+#CUDA_TAG=12.0.1-cudnn8-runtime-ubuntu20.04
+#DAPHNE_TAG=${CUDA_TAG}_$TIMESTAMP_DATE
+#IMAGE_REPO=daphneeu/$DAPHNE_TARGET-cuda
+#BASE_IMAGE=daphneeu/daphne-dev-cuda
+#FINAL_BASE_IMAGE=nvidia/cuda:$CUDA_TAG
+#DAPHNE_BUILD_FLAGS="--mpi --cuda"
+#build_daphne
 
-docker build -t daphneeu/daphne-dev-interactive -f ./daphne-dev-interactive.Dockerfile .
+#-----------------------------------------------------------------------------
+# Images for DAPHNE development (OneAPI)
+#------------------------------------------------------------------------------
+#DAPHNE_TARGET=daphne-dev-interactive
+#ONEAPI_TAG=2023.0.0-devel-ubuntu22.04
+#DAPHNE_TAG=${ONEAPI_TAG}_$TIMESTAMP_DATE
+#IMAGE_REPO=daphneeu/$DAPHNE_TARGET-oneapi
+#BASE_IMAGE=ubuntu:20.04
+#FINAL_BASE_IMAGE=intel/oneapi:$ONEAPI_TAG
+#build_daphne -dev

containers/daphne-dev.Dockerfile

@@ -18,42 +18,76 @@
 # This Dockerfile provides a basic DAPHNE compilation environment with all
 # third party dependencies precompiled (use ''./build.sh --no-deps --installPrefix /usr/local'' to compile DAPHNE)
 
-FROM ubuntu:20.04 as base
+ARG BASE_IMAGE=ubuntu:20.04
+ARG FINAL_BASE_IMAGE=ubuntu:20.04
+ARG CMAKE_VERSION=3.26.2
+ARG TIMESTAMP=0
+
+FROM ${BASE_IMAGE} as base
 ARG DEBIAN_FRONTEND="noninteractive"
 ARG DEBCONF_NOWARNINGS="yes"
 RUN apt-get -qq -y update && apt-get -y upgrade \
     && apt-get -y --no-install-recommends install  \
-    ca-certificates git unzip wget tar \
+    ca-certificates file git unzip wget tar \
     libomp-dev libssl-dev libxml2-dev uuid-dev zlib1g-dev \
     build-essential clang gfortran lld lldb llvm ninja-build openjdk-11-jdk-headless pkg-config \
     && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+FROM base as with-cmake
 ARG NUM_CORES=4
-ARG CMAKE_VERSION=3.25.2
+ARG CMAKE_VERSION
 ARG BUILD_DIR=/build-cmake
 WORKDIR $BUILD_DIR
 RUN wget -qO- https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION.tar.gz | tar xzf \
     - --strip-components=1
 RUN ./bootstrap --parallel=$NUM_CORES --generator=Ninja --no-qt-gui --prefix=/usr/local
+RUN ninja
+RUN strip --strip-unneeded bin/*
 RUN ninja install
 WORKDIR /
 RUN rm -rf ${BUILD_DIR}
 
-FROM base as with-deps
-ARG DAPHNE_DIR=/usr/local/src/daphne
-ARG DAPHNE_REPO=https://github.com/corepointer/daphne.git
-ARG DAPHNE_BRANCH=docker-improvements
+FROM with-cmake as daphne-deps-compile
+ARG DAPHNE_DIR=/daphne
+ARG DAPHNE_REPO=https://github.com/daphne-eu/daphne.git
+ARG DAPHNE_BRANCH=main
+ARG TIMESTAMP
+ARG CREATION_DATE=0
+ARG GIT_HASH=0
+LABEL "org.opencontainers.image.source"="${DAPHNE_REPO}"
+LABEL "org.opencontainers.image.base.name"="${BASE_IMAGE}"
+LABEL "org.opencontainers.image.version"="$TIMESTAMP"
+LABEL "org.opencontainers.image.created"="${CREATION_DATE}"
+LABEL "org.opencontainers.image.revision"="${GIT_HASH}"
 RUN git clone --depth=1 --single-branch --branch=$DAPHNE_BRANCH $DAPHNE_REPO $DAPHNE_DIR
 WORKDIR $DAPHNE_DIR
-RUN ./build.sh -nf --no-submodule-update --installPrefix /usr/local
-WORKDIR /
+RUN ./build.sh --no-fancy --no-submodule-update --installPrefix /usr/local
+RUN find /usr/local -exec file {} \; | grep -e "not stripped" | cut -d ":" -f 1 | xargs strip --strip-unneeded
 RUN rm -rf $DAPHNE_DIR
+RUN ldconfig
+WORKDIR /
 
-FROM ubuntu:20.04 as daphne-dev
+FROM ${FINAL_BASE_IMAGE} as daphne-dev
 ARG DEBIAN_FRONTEND="noninteractive"
-RUN apt-get -qq -y update && apt-get -y upgrade \
-    && apt-get -y --no-install-recommends install  \
-    ca-certificates git unzip wget tar zstd moreutils \
-    libomp-dev libssl-dev libxml2-dev uuid-dev zlib1g-dev \
-    build-essential clang gdb gdbserver gfortran lld lldb llvm ninja-build openjdk-11-jdk-headless pkg-config \
-    python3-numpy && apt-get clean && rm -rf /var/lib/apt/lists/*
-COPY --from=with-deps /usr/local /usr/local
+RUN apt-get -qq -y update && apt-get -y upgrade && apt-get -y --no-install-recommends install  \
+    libssl1.1 libxml2-dev zlib1g-dev libtinfo-dev uuid-dev python3-numpy \
+    build-essential ninja-build openjdk-11-jdk-headless pkg-config \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+COPY --from=daphne-deps-compile /usr/local/bin/ /usr/local/bin/
+COPY --from=daphne-deps-compile /usr/local/include/ /usr/local/include/
+COPY --from=daphne-deps-compile /usr/local/lib/ /usr/local/lib/
+COPY --from=daphne-deps-compile /usr/local/share/ /usr/local/share/
+RUN ldconfig
+
+FROM daphne-dev as daphne-dev-interactive
+ARG DEBIAN_FRONTEND="noninteractive"
+RUN apt-get -qq -y update && apt-get -y upgrade && apt-get -y --no-install-recommends install  \
+    vim nano rsync sudo iputils-ping virtualenv \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+COPY entrypoint-interactive.sh /
+ENTRYPOINT [ "/entrypoint-interactive.sh"]
+
+FROM daphne-dev as github-action
+ARG DEBIAN_FRONTEND="noninteractive"
+RUN apt-get -qq -y update && apt-get -y upgrade && apt-get -y --no-install-recommends install  \
+    moreutils && apt-get clean && rm -rf /var/lib/apt/lists/*

containers/daphne.Dockerfile

@@ -0,0 +1,78 @@
+# syntax=docker/dockerfile:1
+
+# Copyright 2023 The DAPHNE Consortium
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This Dockerfile provides a basic DAPHNE compilation environment with all
+# third party dependencies precompiled (use ''./build.sh --no-deps --installPrefix /usr/local'' to compile DAPHNE)
+
+ARG BASE_IMAGE=daphneeu/daphne-dev
+ARG FINAL_BASE_IMAGE=ubuntu:20.04
+ARG DAPHNE_DIR=/daphne
+ARG DAPHNE_REPO=https://github.com/daphne-eu/daphne.git
+ARG DAPHNE_BRANCH=main
+ARG TIMESTAMP=0
+ARG CREATION_DATE=0
+ARG GIT_HASH=0
+
+FROM ${BASE_IMAGE} as daphne-build
+ARG DAPHNE_DIR
+ARG DAPHNE_REPO
+ARG DAPHNE_BRANCH
+ARG TIMESTAMP
+ARG CREATION_DATE
+ARG GIT_HASH
+ARG DAPHNE_BUILD_FLAGS
+LABEL "org.opencontainers.image.revision"="${DAPHNE_REPO}@${DAPHNE_BRANCH}"
+LABEL "org.opencontainers.image.base.name"="$BASE_IMAGE"
+LABEL "org.opencontainers.image.version"="$TIMESTAMP"
+RUN echo Timestamp: $TIMESTAMP
+RUN echo $TIMESTAMP
+RUN echo ${DAPHNE_REPO}
+RUN echo $DAPHNE_BRANCH
+RUN echo $DAPHNE_DIR 
+RUN git clone --depth=1 --single-branch --branch=$DAPHNE_BRANCH $DAPHNE_REPO $DAPHNE_DIR
+WORKDIR $DAPHNE_DIR
+RUN ldconfig
+RUN ./build.sh --no-fancy --no-deps --no-submodule-update --installPrefix /usr/local $DAPHNE_BUILD_FLAGS
+RUN strip --strip-unneeded bin/*
+RUN strip --strip-unneeded lib/*
+WORKDIR /
+
+FROM ${FINAL_BASE_IMAGE} as daphne
+ARG DAPHNE_DIR
+ARG DAPHNE_REPO
+ARG DAPHNE_BRANCH
+ARG TIMESTAMP
+ARG CREATION_DATE
+ARG GIT_HASH
+LABEL "org.opencontainers.image.source"="${DAPHNE_REPO}"
+LABEL "org.opencontainers.image.base.name"="${BASE_IMAGE}"
+LABEL "org.opencontainers.image.version"="$TIMESTAMP"
+LABEL "org.opencontainers.image.created"="${CREATION_DATE}"
+LABEL "org.opencontainers.image.revision"="${GIT_HASH}"
+RUN apt-get -qq -y update && apt-get -y upgrade && apt-get -y --no-install-recommends install  \
+    libtinfo6 libssl1.1 zlib1g \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+COPY --from=daphne-build $DAPHNE_DIR/bin/* /usr/local/bin
+COPY --from=daphne-build $DAPHNE_DIR/lib/* /usr/local/lib
+COPY --from=daphne-build /usr/local/lib/libparquet.so* /usr/local/lib
+COPY --from=daphne-build /usr/local/lib/libarrow.so* /usr/local/lib
+COPY --from=daphne-build /usr/local/lib/libantlr*.so* /usr/local/lib
+COPY --from=daphne-build /usr/local/lib/libopen*.so* /usr/local/lib
+COPY --from=daphne-build /usr/local/lib/libmpi*.so* /usr/local/lib
+RUN ldconfig
+ENTRYPOINT ["/usr/local/bin/daphne"]
+CMD ["--help"]

containers/entrypoint-interactive.sh

@@ -16,7 +16,7 @@
 
 /usr/sbin/groupadd -g "$GID" dockerusers
 /usr/sbin/useradd -c 'Docker Container User' -u $UID -g "$GID" -G sudo -m -s /bin/bash -d /home/"$USER" "$USER"
-printf "${USER} ALL=(ALL:ALL) NOPASSWD:ALL" | sudo EDITOR="tee -a" visudo >> /dev/null
+printf "${USER} ALL=(ALL:ALL) NOPASSWD:ALL" | sudo EDITOR="tee -a" visudo #>> /dev/null
 touch /home/"$USER"/.sudo_as_admin_successful
 exec su "$USER"
 # set a default password

containers/run-daphne-dev.sh → containers/run-docker-example.sh

@@ -14,12 +14,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+echo "Use this as an example to start DAPHNE docker containers. Copy and customize for the various flavors."
+
 DOCKER_IMAGE=daphneeu/daphne-dev
+#DOCKER_IMAGE=daphneeu/daphne-dev-interactive
+DOCKER_TAG=latest
+#DOCKER_TAG=cuda-12.0.1-cudnn8-devel-ubuntu20.04_2023-04-03
 DAPHNE_ROOT=$PWD
 
-LD_LIBRARY_PATH=/daphne/lib:$LD_LIBRARY_PATH
-PATH=/daphne/bin:$PATH
+CUDA_PATH=/usr/local/cuda
+LD_LIBRARY_PATH=$CUDA_PATH/lib64:$DAPHNE_ROOT/lib:$LD_LIBRARY_PATH
+PATH=$CUDA_PATH/bin:$DAPHNE_ROOT/bin:$PATH
+# uncomment to pass GPU devices to the container
+#DEVICE_FLAGS=--gpus all
 
 # shellcheck disable=SC2046
 # shellcheck disable=SC2068
-docker run --user=$(id -u):$(id -g) --rm -w $DAPHNE_ROOT -e TERM=screen-256color -v "$DAPHNE_ROOT:$DAPHNE_ROOT" $DOCKER_IMAGE $@
+docker run "$DEVICE_FLAGS" --user=$(id -u):$(id -g) --rm -w "$DAPHNE_ROOT" -v "$DAPHNE_ROOT:$DAPHNE_ROOT" \
+    -e TERM=screen-256color -e PATH="$PATH" -e LD_LIBRARY_PATH="$LD_LIBRARY_PATH" -e USER=$(id -n -u) -e UID=$(id -u) \
+    "$DOCKER_IMAGE:$DOCKER_TAG" $@