From 361c976999dfc39f2daffb45bbde2e2a1c86cbe1 Mon Sep 17 00:00:00 2001 From: Sebastien Jourdain Date: Tue, 17 May 2022 16:26:20 -0600 Subject: [PATCH] chore(docker): Update scripts to use v2 www tools --- docker/Dockerfile.common | 53 +++++++++++++ docker/Dockerfile.conda | 26 +++++++ docker/Dockerfile.pip | 36 +++++++++ docker/config/apache/001-trame.conf | 25 +++++++ docker/config/default-launcher.json | 24 ++++++ docker/scripts/conda/activate_venv.sh | 3 + docker/scripts/conda/create_venv.sh | 6 ++ docker/scripts/conda/install_requirements.sh | 5 ++ docker/scripts/entrypoint.sh | 19 +++++ docker/scripts/fix_uid_gid.sh | 32 ++++++++ docker/scripts/generate_launcher_config.py | 78 ++++++++++++++++++++ docker/scripts/generate_www.py | 26 +++++++ docker/scripts/make_directories.sh | 7 ++ docker/scripts/pip/activate_venv.sh | 1 + docker/scripts/pip/create_venv.sh | 5 ++ docker/scripts/pip/install_requirements.sh | 5 ++ docker/scripts/server.sh | 50 +++++++++++++ docker/scripts/start.sh | 71 ++++++++++++++++++ docker/scripts/yaml_to_json.py | 17 +++++ 19 files changed, 489 insertions(+) create mode 100644 docker/Dockerfile.common create mode 100644 docker/Dockerfile.conda create mode 100644 docker/Dockerfile.pip create mode 100644 docker/config/apache/001-trame.conf create mode 100644 docker/config/default-launcher.json create mode 100644 docker/scripts/conda/activate_venv.sh create mode 100644 docker/scripts/conda/create_venv.sh create mode 100644 docker/scripts/conda/install_requirements.sh create mode 100755 docker/scripts/entrypoint.sh create mode 100755 docker/scripts/fix_uid_gid.sh create mode 100755 docker/scripts/generate_launcher_config.py create mode 100755 docker/scripts/generate_www.py create mode 100755 docker/scripts/make_directories.sh create mode 100644 docker/scripts/pip/activate_venv.sh create mode 100644 docker/scripts/pip/create_venv.sh create mode 100644 docker/scripts/pip/install_requirements.sh create mode 100755 docker/scripts/server.sh create mode 100755 docker/scripts/start.sh create mode 100755 docker/scripts/yaml_to_json.py diff --git a/docker/Dockerfile.common b/docker/Dockerfile.common new file mode 100644 index 00000000..1073562d --- /dev/null +++ b/docker/Dockerfile.common @@ -0,0 +1,53 @@ +ARG BASE_IMAGE=ubuntu:20.04 +FROM ${BASE_IMAGE} + +# Necessary to install tzdata. It will default to UTC. +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install -y \ + wget \ + apache2 \ + apache2-dev \ + libapr1-dev \ + apache2-utils \ + gosu && \ + rm -rf /var/lib/apt/lists/* + +# Set up needed permissions and users +RUN groupadd trame-user -g 1000 && \ + groupadd proxy-mapping -g 1001 && \ + useradd -u 1000 -g trame-user -G proxy-mapping -s /sbin/nologin trame-user && \ + usermod -a -G proxy-mapping www-data && \ + mkdir -p /opt/trame && \ + chown -R trame-user:trame-user /opt/trame && \ + mkdir -p /home/trame-user && \ + chown -R trame-user:trame-user /home/trame-user && \ + touch /opt/trame/proxy-mapping.txt && \ + chown trame-user:proxy-mapping /opt/trame/proxy-mapping.txt && \ + chmod 660 /opt/trame/proxy-mapping.txt && \ + mkdir -p /deploy && \ + chown -R trame-user:trame-user /deploy + +# Copy the apache configuration file into place +COPY config/apache/001-trame.conf /etc/apache2/sites-available/001-trame.conf +COPY config/default-launcher.json /opt/trame/default-launcher.json + +# Configure the apache web server +RUN a2enmod vhost_alias && \ + a2enmod proxy && \ + a2enmod proxy_http && \ + a2enmod proxy_wstunnel && \ + a2enmod rewrite && \ + a2enmod headers && \ + a2dissite 000-default.conf && \ + a2ensite 001-trame && \ + a2dismod autoindex -f + +# Copy the scripts into place +COPY scripts/* /opt/trame/ + +# Open port 80 to the world outside the container +EXPOSE 80 + +ENTRYPOINT ["/opt/trame/entrypoint.sh"] diff --git a/docker/Dockerfile.conda b/docker/Dockerfile.conda new file mode 100644 index 00000000..aa59ad8d --- /dev/null +++ b/docker/Dockerfile.conda @@ -0,0 +1,26 @@ +ARG BASE_IMAGE=trame-common +FROM ${BASE_IMAGE} + +# Install miniconda +ENV CONDA_DIR /opt/conda +RUN if [ $(uname -m) = "x86_64" ]; then arch="x86_64"; else arch="aarch64"; fi && \ + wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-$arch.sh -O /miniconda.sh && \ + /bin/bash /miniconda.sh -b -p $CONDA_DIR && \ + rm /miniconda.sh && \ + chown -R trame-user:trame-user $CONDA_DIR + +# Put conda in the path +ENV PATH=$CONDA_DIR/bin:$PATH + +# Install pyyaml +RUN gosu trame-user conda install -y --freeze-installed -c conda-forge \ + pyyaml && \ + conda clean -afy + +# Copy the scripts into place +COPY scripts/conda/* /opt/trame/ + +# Set venv paths +ENV TRAME_VENV=/deploy/server/venv +ENV PV_VENV=$TRAME_VENV +ENV VTK_VENV=$TRAME_VENV diff --git a/docker/Dockerfile.pip b/docker/Dockerfile.pip new file mode 100644 index 00000000..946ef6c2 --- /dev/null +++ b/docker/Dockerfile.pip @@ -0,0 +1,36 @@ +ARG BASE_IMAGE=trame-common +FROM ${BASE_IMAGE} + +RUN apt-get update && \ + apt-get install -y \ + python3.9 \ + # python3.9-distutils is required to install pip + # it unfortunately has to install python3.8-minimal as well... + python3.9-distutils \ + # python-is-python3 creates a symlink for python to python3 + python-is-python3 \ + # For creating virtual environments + python3.9-venv && \ + rm -rf /var/lib/apt/lists/* + +# Set python3 to python3.9 (otherwise, it will be python3.8) +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1 + +# Never use a cache directory for pip, both here in this Dockerfile +# and when we run the container. +ENV PIP_NO_CACHE_DIR=1 + +# Install and upgrade pip +RUN wget -q -O- https://bootstrap.pypa.io/get-pip.py | python3.9 && \ + pip install -U pip + +# Install setup dependencies +RUN pip install PyYAML + +# Copy the pip scripts into place +COPY scripts/pip/* /opt/trame/ + +# Set venv paths +ENV TRAME_VENV=/deploy/server/venv +ENV PV_VENV=$TRAME_VENV +ENV VTK_VENV=$TRAME_VENV diff --git a/docker/config/apache/001-trame.conf b/docker/config/apache/001-trame.conf new file mode 100644 index 00000000..d0c0d39d --- /dev/null +++ b/docker/config/apache/001-trame.conf @@ -0,0 +1,25 @@ + + DocumentRoot /deploy/server/www + ErrorLog /deploy/server/logs/apache/error.log + CustomLog /deploy/server/logs/apache/access.log combined + + + Options Indexes FollowSymLinks + Order allow,deny + Allow from all + AllowOverride None + Require all granted + + + Header set Access-Control-Allow-Origin "*" + + # Handle launcher forwarding + ProxyPass /launcher http://localhost:9000/paraview + ProxyPass /paraview http://localhost:9000/paraview + + # Handle WebSocket forwarding + RewriteEngine On + RewriteMap session-to-port txt:/opt/trame/proxy-mapping.txt + RewriteCond %{QUERY_STRING} ^sessionId=(.*)&path=(.*)$ [NC] + RewriteRule ^/proxy.*$ ws://${session-to-port:%1}/%2 [P] + diff --git a/docker/config/default-launcher.json b/docker/config/default-launcher.json new file mode 100644 index 00000000..265c3f66 --- /dev/null +++ b/docker/config/default-launcher.json @@ -0,0 +1,24 @@ +{ + "configuration": { + "host": "0.0.0.0", + "port": 9000, + "endpoint": "paraview", + "log_dir": "/deploy/server/logs/launcher", + "proxy_file": "/opt/trame/proxy-mapping.txt", + "sessionURL": "ws://USE_HOST/proxy?sessionId=${id}&path=ws", + "timeout": 25, + "sanitize": {}, + "fields": [] + }, + "resources": [ + { + "host": "0.0.0.0", + "port_range": [ + 9001, + 9500 + ] + } + ], + "properties": {}, + "apps": {} +} diff --git a/docker/scripts/conda/activate_venv.sh b/docker/scripts/conda/activate_venv.sh new file mode 100644 index 00000000..ebc226fd --- /dev/null +++ b/docker/scripts/conda/activate_venv.sh @@ -0,0 +1,3 @@ +# Ensure we are in the base conda environment first, or it won't work. +. $CONDA_DIR/bin/activate +conda activate $TRAME_VENV diff --git a/docker/scripts/conda/create_venv.sh b/docker/scripts/conda/create_venv.sh new file mode 100644 index 00000000..6dd3eb73 --- /dev/null +++ b/docker/scripts/conda/create_venv.sh @@ -0,0 +1,6 @@ +# Ensure we are in the base conda environment first, or it won't work. +. $CONDA_DIR/bin/activate + +conda create -y conda python=3.9 --prefix $TRAME_VENV +conda activate $TRAME_VENV +conda clean -afy diff --git a/docker/scripts/conda/install_requirements.sh b/docker/scripts/conda/install_requirements.sh new file mode 100644 index 00000000..1cd88070 --- /dev/null +++ b/docker/scripts/conda/install_requirements.sh @@ -0,0 +1,5 @@ +# Install requirements (if it exists) +if [ -f /deploy/setup/requirements.txt ] +then + conda install -y --file /deploy/setup/requirements.txt +fi diff --git a/docker/scripts/entrypoint.sh b/docker/scripts/entrypoint.sh new file mode 100755 index 00000000..5479ce60 --- /dev/null +++ b/docker/scripts/entrypoint.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +if [ ! -d /deploy/server ] && [ ! -d /deploy/setup ] +then + echo "ERROR: The deploy directory must be mounted into the container at /deploy" + exit 1 +fi + +# Fix any uid/gid mismatch +/opt/trame/fix_uid_gid.sh + +# Ensure the needed directories exist +gosu trame-user /opt/trame/make_directories.sh + +# Restart apache +service apache2 restart + +# Start the server +gosu trame-user /opt/trame/server.sh diff --git a/docker/scripts/fix_uid_gid.sh b/docker/scripts/fix_uid_gid.sh new file mode 100755 index 00000000..01a7f231 --- /dev/null +++ b/docker/scripts/fix_uid_gid.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +check_dir=/deploy/server +if [ ! -d $check_dir ] +then + check_dir=/deploy/setup +fi + +deploy_uid=$(stat -c '%u' $check_dir) +deploy_gid=$(stat -c '%g' $check_dir) + +trame_user_uid=$(id -u trame-user) +trame_user_gid=$(id -g trame-user) + +run_chown=false +if [[ "$deploy_uid" != "$trame_user_uid" ]]; then + usermod --uid $deploy_uid trame-user + run_chown=true +fi + +if [[ "$deploy_gid" != "$trame_user_gid" ]]; then + groupmod --gid $deploy_gid trame-user + run_chown=true +fi + +if [[ "$run_chown" == false ]]; then + exit 0 +fi + +# Run chown on all trame-user directories/files +chown -R trame-user:trame-user /opt/trame +chown trame-user:proxy-mapping /opt/trame/proxy-mapping.txt diff --git a/docker/scripts/generate_launcher_config.py b/docker/scripts/generate_launcher_config.py new file mode 100755 index 00000000..0496140e --- /dev/null +++ b/docker/scripts/generate_launcher_config.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 + +import json +from pathlib import Path + + +def run(input_path, apps_path, out_path): + default_command_flags = [ + '--host', '${host}', + '--port', '${port}', + '--authKey', '${secret}', + '--server', + ] + default_ready_line = 'Starting factory' + + with open(input_path, 'r') as rf: + input_dict = json.load(rf) + + if not Path(apps_path).exists(): + raise Exception(f'{apps_path} does not exist') + + with open(apps_path, 'r') as rf: + apps_dict = json.load(rf) + + if not apps_dict: + raise Exception(f'{apps_path} is empty') + + for app_name, config in apps_dict.items(): + if 'app' not in config: + msg = ( + f'In {apps_path}, every app must contain an "app" key, but ' + f'"{app_name}" does not' + ) + raise Exception(msg) + + if 'cmd' in config and 'args' in config: + msg = ( + f'In {apps_path}, "args" and "cmd" cannot both be specified. ' + '"args" is for appending extra args to the default "cmd", but ' + '"cmd" is for overriding the command entirely. Error occurred ' + f'in "{app_name}".' + ) + raise Exception(msg) + + app = config['app'] + default_cmd = [app] + default_command_flags + cmd = config.get('cmd', default_cmd) + cmd += config.get('args', []) + ready_line = config.get('ready_line', default_ready_line) + + input_dict.setdefault('apps', {}) + input_dict['apps'][app_name] = { + 'cmd': cmd, + 'ready_line': ready_line, + } + + if 'trame' not in input_dict['apps']: + # Make a copy of the first app and put it in PyWebVue, so that + # the default localhost:9000 web page will use that app. + first_key = next(iter(input_dict['apps'])) + input_dict['apps']['trame'] = input_dict['apps'][first_key] + + with open(out_path, 'w') as wf: + json.dump(input_dict, wf, indent=2) + + +if __name__ == '__main__': + default_input_path = '/opt/trame/default-launcher.json' + override_input_path = '/deploy/setup/launcher.json' + apps_path = '/opt/trame/apps.json' + out_path = '/deploy/server/launcher.json' + + if Path(override_input_path).exists(): + input_path = override_input_path + else: + input_path = default_input_path + + run(input_path, apps_path, out_path) diff --git a/docker/scripts/generate_www.py b/docker/scripts/generate_www.py new file mode 100755 index 00000000..5c2a27ea --- /dev/null +++ b/docker/scripts/generate_www.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python + +import json +import subprocess + + +def run(apps_path, out_path, *modules): + # generate www content + cmd = ["python", "-m", "trame.tools.www", "--output", f'{out_path}', *modules] + subprocess.run(cmd) + + with open(apps_path, 'r') as rf: + apps_dict = json.load(rf) + + # FIXME need to generate index.html => {app_name}.html + # for app_name, config in apps_dict.items(): + # name = config['app'] + # => cp out_path/index.html out_path/{name}.html + # => replace data-app-name="trame" => data-app-name="{name}" + + +if __name__ == '__main__': + apps_path = '/opt/trame/apps.json' + out_path = '/deploy/server/www' + modules = ["client", "trame", "vtk", "vuetify", "plotly", "router", "vega", "markdown"] + run(apps_path, out_path, *modules) diff --git a/docker/scripts/make_directories.sh b/docker/scripts/make_directories.sh new file mode 100755 index 00000000..880c13f1 --- /dev/null +++ b/docker/scripts/make_directories.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +# Make directories we need before running the rest of the scripts +mkdir -p \ + /deploy/server/www \ + /deploy/server/logs/apache \ + /deploy/server/logs/launcher diff --git a/docker/scripts/pip/activate_venv.sh b/docker/scripts/pip/activate_venv.sh new file mode 100644 index 00000000..19296a55 --- /dev/null +++ b/docker/scripts/pip/activate_venv.sh @@ -0,0 +1 @@ +. $TRAME_VENV/bin/activate diff --git a/docker/scripts/pip/create_venv.sh b/docker/scripts/pip/create_venv.sh new file mode 100644 index 00000000..5fc6187a --- /dev/null +++ b/docker/scripts/pip/create_venv.sh @@ -0,0 +1,5 @@ +python -m venv $TRAME_VENV +. $TRAME_VENV/bin/activate + +# Update pip +pip install -U pip diff --git a/docker/scripts/pip/install_requirements.sh b/docker/scripts/pip/install_requirements.sh new file mode 100644 index 00000000..7e79cfce --- /dev/null +++ b/docker/scripts/pip/install_requirements.sh @@ -0,0 +1,5 @@ +# Install requirements (if it exists) +if [ -f /deploy/setup/requirements.txt ] +then + pip install -r /deploy/setup/requirements.txt +fi diff --git a/docker/scripts/server.sh b/docker/scripts/server.sh new file mode 100755 index 00000000..44f678f4 --- /dev/null +++ b/docker/scripts/server.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +if [ ! -d $TRAME_VENV ] +then + # We have access to PyYAML in the root python environment. + # Convert the apps.yml file to json and put it in the right place. + python /opt/trame/yaml_to_json.py /deploy/setup/apps.yml /opt/trame/apps.json + + # Create (and activate) the venv + . /opt/trame/create_venv.sh + + # Run the initialize script (if it exists) + if [ -f /deploy/setup/initialize.sh ] + then + . /deploy/setup/initialize.sh + fi + + # Install any specified requirements + . /opt/trame/install_requirements.sh + + # Generate launcher.json and the www directory + python /opt/trame/generate_launcher_config.py + python /opt/trame/generate_www.py + + # Merge any user-created www directories with the generated one + if [ -d /deploy/setup/www ] + then + cp -r /deploy/setup/www/* /deploy/server/www + fi +else + . /opt/trame/activate_venv.sh +fi + +if [[ $TRAME_BUILD_ONLY == 1 ]]; then + echo "Build complete. Exiting." + exit 0 +fi + +# Copy the launcher config into the location where the start script expects +# to find it. The config may or may not have replacement values in it, if it +# does not, the start script will not change it in any way. Here we expect +# that the user doing the "docker run ..." has set up an external directory +# containing a "server/launcher.json" filepath and mounts that path as +# "/deploy". +cp /deploy/server/launcher.json /opt/trame/config-template.json + +# This performs replacements on the launcher-template.json copied into place +# above, based on the presence of environment variables passed with "-e" to the +# "docker run ..." command. +. /opt/trame/start.sh diff --git a/docker/scripts/start.sh b/docker/scripts/start.sh new file mode 100755 index 00000000..b7c01bc0 --- /dev/null +++ b/docker/scripts/start.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +# +# Patches launcher configuration session url, as well as perhaps any +# additional arguments to python, then restarts the apache webserver +# and starts the launcher in the foreground. You can optionally pass a +# custom session root url (e.g: "wss://www.example.com") which will be +# used instead of the default. +# +# You can also pass extra arguments after the session url that will be +# provided as extra arguments to python. In this case, you must also +# pass the session url argument first. +# +# Examples +# +# To just accept the defaults of "ws://localhost": +# +# ./start.sh +# +# To choose 'wss' and 'www.customhost.com', set the following two environment +# variables before invoking this script: +# +# export SERVER_NAME="'www.customhost.com" +# export PROTOCOL="wss" +# +# You can also pass any extra args to python in an environment variable +# as follows: +# +# export EXTRA_PVPYTHON_ARGS="-dr,--mesa-swr" +# +# Note that the extra args to be passed to python should be separated by +# commas, and no extra spaces are used. +# +# When this script is used as the entrypoint in a Docker image, the environment +# variables can be provided using as many "-e" arguments to the "docker run..." +# command as necessary: +# +# docker run --rm \ +# -e SERVER_NAME="www.customhost.com" \ +# -e PROTOCOL="wss" +# -e EXTRA_PVPYTHON_ARGS="-dr,--mesa-swr" \ +# ... +# + +ROOT_URL="ws://localhost" +REPLACEMENT_ARGS="" + +LAUNCHER_TEMPLATE_PATH=/opt/trame/config-template.json +LAUNCHER_PATH=/opt/trame/config.json + +if [[ ! -z "${SERVER_NAME}" ]] && [[ ! -z "${PROTOCOL}" ]] +then + ROOT_URL="${PROTOCOL}://${SERVER_NAME}" +fi + +if [[ ! -z "${EXTRA_PVPYTHON_ARGS}" ]] +then + IFS=',' read -ra EXTRA_ARGS <<< "${EXTRA_PVPYTHON_ARGS}" + for arg in "${EXTRA_ARGS[@]}"; do + REPLACEMENT_ARGS="${REPLACEMENT_ARGS}\"$arg\", " + done +fi + +INPUT=$(<"${LAUNCHER_TEMPLATE_PATH}") +OUTPUT="${INPUT//"SESSION_URL_ROOT"/$ROOT_URL}" +OUTPUT="${OUTPUT//"EXTRA_PVPYTHON_ARGS"/$REPLACEMENT_ARGS}" +echo -e "$OUTPUT" > "${LAUNCHER_PATH}" + +# Run the pvw launcher in the foreground so this script doesn't end +echo "Starting the wslink launcher at" +python -m wslink.launcher ${LAUNCHER_PATH} diff --git a/docker/scripts/yaml_to_json.py b/docker/scripts/yaml_to_json.py new file mode 100755 index 00000000..ff30ad5f --- /dev/null +++ b/docker/scripts/yaml_to_json.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 + +import json +import sys +import yaml + +if len(sys.argv) < 3: + sys.exit('Usage: