addons: add dynamic mount examples (#4148)

* Add base image for fluid dynamic mount feature

Signed-off-by: trafalgarzzz <[email protected]>

* Add juicefs examples for fluid dynamic mount feature

Signed-off-by: trafalgarzzz <[email protected]>

* clean up mount point unconditionally

Signed-off-by: trafalgarzzz <[email protected]>

* Dump mount point logs to /var/log/fluid

Signed-off-by: trafalgarzzz <[email protected]>

* Anchor base image for dynamic mount example

Signed-off-by: trafalgarzzz <[email protected]>

---------

Signed-off-by: trafalgarzzz <[email protected]>

addons/dynamic-mount/base/Dockerfile

@@ -0,0 +1,25 @@
+FROM debian:bullseye@sha256:a165446a88794db4fec31e35e9441433f9552ae048fb1ed26df352d2b537cb96 as builder
+
+RUN apt update && apt install -y build-essential libfuse3-dev pkg-config git python3-pip
+
+RUN pip install meson ninja
+
+RUN git clone https://github.com/libfuse/libfuse.git
+
+RUN mkdir -p libfuse/build && cd libfuse/build && meson setup .. && ninja install
+
+RUN cd libfuse/example && gcc -Wall passthrough.c `pkg-config fuse3 --cflags --libs` -o passthrough
+
+FROM debian:bullseye-slim@sha256:a165446a88794db4fec31e35e9441433f9552ae048fb1ed26df352d2b537cb96
+
+RUN apt update && apt install -y python3 fuse tini supervisor inotify-tools jq && rm -rf /var/cache/apt/* && ln -s /usr/bin/python3 /usr/local/bin/python
+COPY inotify-fluid-config.ini /tmp/inotify-fluid-config.ini
+RUN cat /tmp/inotify-fluid-config.ini >> /etc/supervisor/supervisord.conf && rm /tmp/inotify-fluid-config.ini
+
+COPY reconcile_mount_program_settings.py mount-helper.sh inotify.sh mount-passthrough-fuse.sh prestop.sh entrypoint.sh /usr/local/bin/
+RUN chmod u+x /usr/local/bin/mount-helper.sh /usr/local/bin/inotify.sh /usr/local/bin/mount-passthrough-fuse.sh /usr/local/bin/prestop.sh /usr/local/bin/entrypoint.sh
+
+RUN apt update && apt install -y libfuse3-3 fuse3
+COPY --from=builder libfuse/example/passthrough /usr/local/bin/passthrough
+
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

addons/dynamic-mount/base/build.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set +x
+
+docker build . --network=host -f Dockerfile -t fluidcloudnative/dynamic-mount:base
+
+docker push fluidcloudnative/dynamic-mount:base

addons/dynamic-mount/base/entrypoint.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e
+
+trap "/usr/local/bin/prestop.sh" SIGTERM 
+
+if [[ "$USE_PASSTHROUGH_FUSE" == "True" ]]; then
+mkdir -p $MOUNT_POINT
+cat << EOF >> /etc/supervisor/supervisord.conf
+
+[program:passthrough-fuse]
+command=/usr/local/bin/mount-passthrough-fuse.sh
+redirect_stderr=true
+stdout_logfile=/proc/1/fd/1
+stdout_logfile_maxbytes=0
+autorestart=true
+startretries=9999
+EOF
+fi
+
+supervisord -n

addons/dynamic-mount/base/inotify-fluid-config.ini

@@ -0,0 +1,7 @@
+[program:inotify-fluid-config]
+command=/usr/local/bin/inotify.sh
+redirect_stderr=true
+stdout_logfile=/proc/1/fd/1
+stdout_logfile_maxbytes=0
+autorestart=true
+startretries=9999

addons/dynamic-mount/base/inotify.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -xe
+
+FUSE_CONFIG="/etc/fluid/config"
+
+python /usr/local/bin/reconcile_mount_program_settings.py
+supervisorctl update
+
+# if fuse-config(/etc/fluid/config/config.json) is modified, reconcile setting files under /etc/supervisor.d and use `supervisorctl update` to start/stop new/old fuse daemon process.
+# config.json is mounted by configmap, it is actually a symlink point to actual file, and kubernetes would atomically rename ..data_tmp to ..data, which triggers an inotify moved_to event.
+# Please see https://github.com/kubernetes/kubernetes/blob/master/pkg/volume/util/atomic_writer.go#L93-L138 for more information
+inotifywait -m -r -e moved_to "${FUSE_CONFIG}" |
+    while read -r directory event file; do
+        echo "${directory} ${file} changed (event: ${event})"
+        # mount_and_umount
+        python /usr/local/bin/reconcile_mount_program_settings.py 
+        supervisorctl update
+    done

addons/dynamic-mount/base/mount-helper.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+set -ex
+
+function help() {
+    echo "Usage: "
+    echo "  bash mount-helper.sh mount|umount [args...]"
+    echo "Examples: "
+    echo "  1. mount filesystem [mount_src] to [mount_target] with options defined in [mount_opt_file]"
+    echo "     bash mount-helper.sh mount [mount_src] [mount_target] [mount_opt_file]"
+    echo "  2. umount filesystem mounted at [mount_target]"
+    echo "     bash mount-helper.sh umount [mount_target]"
+}
+
+function error_msg() {
+    help
+    echo
+    echo $1
+    exit 1
+}
+
+function clean_up() {
+    # Ignore any possible error in clean up process
+    set +e 
+    mount_target=$1
+    if [[ -z "$mount_target" ]]; then
+        return
+    fi
+    umount $mount_target
+    sleep 3 # umount may be asynchronous
+    rmdir $mount_target
+}
+
+function mount_fn() {
+    if [[ $# -ne 4 ]]; then
+        error_msg "Error: mount-helper.sh mount expects 4 arguments, but got $# arguments."
+    fi
+    mount_src=$1
+    mount_target=$2
+    fs_type=$3
+    mount_opt_file=$4
+
+    # NOTES.1: umount $mount_target here to avoid [[ -d $mount_target ]] returning "Transport Endpoint is not connected" error.
+    # NOTES.2: Use "cat /proc/self/mountinfo" instead of the "mount" command because Alpine has some issue on printing mount info with "mount".
+    if cat /proc/self/mountinfo | grep " ${mount_target} " > /dev/null; then
+        echo "found mount point on ${mount_target}, umount it before re-mount."
+        umount ${mount_target}
+    fi
+
+    if [[ ! -d "$mount_target" ]]; then
+        mkdir -p "$mount_target"
+    fi
+
+    # mount-helper.sh should be wrapped in `tini -s -g` so trap will be triggered
+    trap "clean_up $mount_target" SIGTERM EXIT
+    /opt/mount.sh $mount_src $mount_target $fs_type $mount_opt_file
+}
+
+function umount_fn() {
+    if [[ $# -ne 1 ]]; then
+        error_msg "Error: mount-helper.sh umount expects 1 argument, but got $# arguments."
+    fi
+    umount $1 || true
+}
+
+function main() {
+    if [[ $# -eq 0 ]]; then
+        error_msg "Error: not enough arguments, require at least 1 argument"
+    fi
+
+    if [[ $# -gt 0 ]]; then
+        case $1 in
+            mount)
+                shift
+                mount_fn $@
+                ;;
+            unmount|umount)
+                shift
+                umount_fn $@
+                ;;
+            *)
+                error_msg "Error: unknown option: $1"
+                ;;
+        esac
+    fi 
+}
+
+main $@
+

addons/dynamic-mount/base/mount-passthrough-fuse.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -ex
+
+umount $MOUNT_POINT || true
+passthrough -o modules=subdir,subdir=/mnt,auto_unmount -f $MOUNT_POINT

addons/dynamic-mount/base/prestop.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -e
+
+mount_points=$(cat /proc/self/mountinfo | grep " ${MOUNT_POINT}" | awk '{print $5}')
+
+echo "prestop.sh: umounting mountpoints under ${MOUNT_POINT}"
+for mount_point in ${mount_points}; do
+    echo ">> mount-helper.sh umount ${mount_point}"
+    mount-helper.sh umount ${mount_point}
+done
+
+# from now on, we clean sub dirs in a best-effort manner.
+set +e
+echo "prestop.sh: clean sub directories under ${MOUNT_POINT}"
+sub_dirs=$(ls "${MOUNT_POINT}/")
+for sub_dir in ${sub_dirs}; do
+    rmdir "${MOUNT_POINT}/${sub_dir}" || echo "WARNING: failed to rmdir ${sub_dir}, maybe filesystem still mounting on it."
+done
+
+exit 0

addons/dynamic-mount/base/reconcile_mount_program_settings.py

@@ -0,0 +1,90 @@
+import json
+import glob
+import os
+
+USE_PASSTHROUGH_FUSE = os.environ.get("USE_PASSTHROUGH_FUSE", 'False') == 'True'
+
+FLUID_RUNTIME_MNT = os.environ.get("MOUNT_POINT")
+FLUID_MOUNT_OPT_DIR = "/etc/fluid/mount-opts"
+FLUID_CONFIG_FILE = "/etc/fluid/config/config.json"
+SUPERVISORD_SETTING_DIR = "/etc/supervisor/conf.d"
+SUPERVISORD_SETTING_TEMPLATE = """[program:{name}]
+command=tini -s -g -- mount-helper.sh mount {mount_src} {mount_target} {fs_type} {mount_opt_file}
+stdout_logfile=/var/log/fluid/{name}.out
+stderr_logfile=/var/log/fluid/{name}.err
+autorestart=true
+startretries=9999"""
+
+def prepare_dirs():
+    os.makedirs(SUPERVISORD_SETTING_DIR, exist_ok=True)
+    os.makedirs("/var/log/fluid", exist_ok=True)
+    os.makedirs(FLUID_MOUNT_OPT_DIR, exist_ok=True)
+
+def write_mount_opts(mount_opts, opt_file):
+    with open(opt_file, "w") as f:
+        f.write(json.dumps(mount_opts))
+
+def reconcile_supervisord_settings():
+    rawStr = ""
+    with open(FLUID_CONFIG_FILE, "r") as f:
+        rawStr = f.readlines()
+
+    print(f"{FLUID_CONFIG_FILE}: {rawStr[0]}") # config.json only have one line in json format
+
+    setting_files = glob.glob(os.path.join(SUPERVISORD_SETTING_DIR, "*.conf"))
+
+    # obj["mounts"] is like [{"mountPoint": "s3://mybucket", "name": "mybucket", "path": "/mybucket", "options":{...}}, {"mountPoint": "s3://mybucket2", "name": "mybucket2", "path": "/mybucket2", "options":{...}}]
+    obj = json.loads(rawStr[0])
+    expected_mounts = [mount["name"] for mount in obj["mounts"]]
+    current_mounts = [os.path.basename(file).removesuffix(".conf") for file in setting_files]
+
+    need_mount = list(set(expected_mounts).difference(set(current_mounts)))
+    need_unmount = list(set(current_mounts).difference(set(expected_mounts)))
+    print(f"need mount: {need_mount}, need umount: {need_unmount}")
+
+    for name in need_unmount:
+        setting_file = os.path.join(SUPERVISORD_SETTING_DIR, f"{name}.conf")
+        if os.path.isfile(setting_file):
+            os.remove(setting_file)
+            print(f"Mount \"{name}\"'s settings has been removed.")
+
+
+    access_mode = "ro"
+    if "ReadWriteMany" in obj["accessModes"]:
+        access_mode = "rw"
+    mount_info_dict = {mount["name"]: mount for mount in obj["mounts"]}
+    for name in need_mount:
+        if name not in mount_info_dict:
+            print(f"WARNING: mount \"{name}\" is not found in {FLUID_CONFIG_FILE}.")
+            continue
+        mount_info = mount_info_dict[name]
+        mount_src: str = mount_info["mountPoint"]
+        fs_type = "unknown"
+        if len(mount_src.split("://")) == 2:
+            fs_type = mount_src.split("://")[0] # e.g. mount_src="nfs://xxxx/yyyy" => fs_type=nfs
+        mount_dir_name = name
+        if "path" in mount_info:
+            if mount_info["path"] != "/":
+                mount_dir_name = mount_info["path"].lstrip("/")
+            else:
+                print(f"WARNING: mounting \"{name}\" at \"/\" is not allowed, fall back to mount at \"/{name}\"")
+        if USE_PASSTHROUGH_FUSE:
+            mount_target = os.path.join("/mnt", mount_dir_name)
+        else:
+            mount_target = os.path.join(FLUID_RUNTIME_MNT, mount_dir_name)
+        mount_opt_file = os.path.join(FLUID_MOUNT_OPT_DIR, f"{name}.opts")
+
+        mount_opts = mount_info["options"]
+        mount_opts["name"] = name
+        mount_opts["access_mode"] = access_mode
+        write_mount_opts(mount_opts, mount_opt_file)
+
+        setting_file = os.path.join(SUPERVISORD_SETTING_DIR, f"{name}.conf")
+        with open(setting_file, 'w') as f:
+            f.write(SUPERVISORD_SETTING_TEMPLATE.format(name=name, mount_src=mount_src, mount_target=mount_target, fs_type=fs_type, mount_opt_file=mount_opt_file))
+
+        print(f"Mount \"{name}\"'s setting is successfully written to {setting_file}")
+
+if __name__=="__main__":
+    prepare_dirs()
+    reconcile_supervisord_settings()

addons/dynamic-mount/juicefs/docker/Dockerfile.juicefs

@@ -0,0 +1,29 @@
+FROM fluidcloudnative/fluid-dynamic-mount-base:v0.4
+
+# Install Juicefs
+WORKDIR /app
+
+ARG TARGETARCH
+ENV JUICEFS_CLI=/usr/bin/juicefs
+ENV JFS_MOUNT_PATH=/usr/local/juicefs/mount/jfsmount
+
+RUN apt update && apt install -y software-properties-common wget gnupg gnupg2 && bash -c "if [[ '${TARGETARCH}' == amd64 ]]; then wget -O - https://download.gluster.org/pub/gluster/glusterfs/10/rsa.pub | apt-key add - && \
+    echo deb [arch=${TARGETARCH}] https://download.gluster.org/pub/gluster/glusterfs/10/LATEST/Debian/buster/${TARGETARCH}/apt buster main > /etc/apt/sources.list.d/gluster.list && \
+    apt-get update && apt-get install -y uuid-dev libglusterfs-dev glusterfs-common; fi"
+
+RUN apt-get update && apt-get install -y librados2 curl fuse procps iputils-ping strace iproute2 net-tools tcpdump lsof librados-dev libcephfs-dev librbd-dev && \
+    rm -rf /var/cache/apt/* && \
+    bash -c "curl -o ${JUICEFS_CLI} https://juicefs.com/static/juicefs.4.9 && \
+    chmod a+x ${JUICEFS_CLI} && mkdir -p /usr/local/juicefs/mount && curl -o ${JFS_MOUNT_PATH} https://juicefs.com/static/Linux/mount.4.9 && chmod a+x ${JFS_MOUNT_PATH};" && \
+    chmod +x ${JUICEFS_CLI} && \
+    mkdir -p /root/.juicefs && \
+    ln -s /usr/local/bin/python /usr/bin/python && \
+    mkdir /root/.acl && cp /etc/passwd /root/.acl/passwd && cp /etc/group /root/.acl/group && \
+    ln -sf /root/.acl/passwd /etc/passwd && ln -sf /root/.acl/group  /etc/group
+
+RUN /usr/bin/juicefs version
+
+# Install mount script for dynamic mount
+RUN apt install -y jq 
+COPY mount.sh /opt/mount.sh
+RUN chmod u+x /opt/mount.sh

addons/dynamic-mount/juicefs/docker/mount.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+
+if [[ $# -ne 4 ]]; then
+    echo "Error: require 3 arguments, but got $# arguments"
+    exit 1
+fi
+
+mount_src=$1        # e.g. juicefs://mybucket
+mount_target=$2     # e.g. /runtime-mnt/thin/default/thin-demo/thin-fuse/mybucket
+fs_type=$3
+mount_opt_file=$4   # e.g. /etc/fluid/mount-opts/mybucket.opts (mount options in json format)
+
+filesystem_name=${mount_src#juicefs://}
+token_file=$(cat ${mount_opt_file} | jq -r '.["token"]')
+access_key_file=$(cat ${mount_opt_file} | jq -r '.["access-key"]')
+secret_key_file=$(cat ${mount_opt_file} | jq -r '.["secret-key"]')
+bucket=$(cat ${mount_opt_file} | jq -r '.["bucket"]')
+
+juicefs auth $filesystem_name --token `cat $token_file` --access-key `cat $access_key_file` --secret-key `cat $secret_key_file` --bucket "$bucket"
+
+exec juicefs mount -f $filesystem_name $mount_target