From a28b411b3921026f1daae64b6acd8249c07718ed Mon Sep 17 00:00:00 2001 From: trafalgarzzz Date: Fri, 31 May 2024 15:55:49 +0800 Subject: [PATCH 1/5] Add base image for fluid dynamic mount feature Signed-off-by: trafalgarzzz --- addons/dynamic-mount/base/Dockerfile | 29 ++++++ addons/dynamic-mount/base/build.sh | 6 ++ addons/dynamic-mount/base/entrypoint.sh | 21 +++++ .../base/inotify-fluid-config.ini | 7 ++ addons/dynamic-mount/base/inotify.sh | 18 ++++ addons/dynamic-mount/base/mount-helper.sh | 88 ++++++++++++++++++ .../base/mount-passthrough-fuse.sh | 5 ++ addons/dynamic-mount/base/prestop.sh | 20 +++++ .../base/reconcile_mount_program_settings.py | 90 +++++++++++++++++++ 9 files changed, 284 insertions(+) create mode 100644 addons/dynamic-mount/base/Dockerfile create mode 100644 addons/dynamic-mount/base/build.sh create mode 100644 addons/dynamic-mount/base/entrypoint.sh create mode 100644 addons/dynamic-mount/base/inotify-fluid-config.ini create mode 100644 addons/dynamic-mount/base/inotify.sh create mode 100644 addons/dynamic-mount/base/mount-helper.sh create mode 100644 addons/dynamic-mount/base/mount-passthrough-fuse.sh create mode 100644 addons/dynamic-mount/base/prestop.sh create mode 100644 addons/dynamic-mount/base/reconcile_mount_program_settings.py diff --git a/addons/dynamic-mount/base/Dockerfile b/addons/dynamic-mount/base/Dockerfile new file mode 100644 index 00000000000..b4b2ed70d8e --- /dev/null +++ b/addons/dynamic-mount/base/Dockerfile @@ -0,0 +1,29 @@ +FROM debian:bullseye as builder + +RUN sed -i 's/deb.debian.org/mirrors.aliyun.com/g' /etc/apt/sources.list + +RUN apt update && apt install -y build-essential libfuse3-dev pkg-config git python3-pip + +RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple + +RUN pip install meson ninja + +RUN git clone https://github.com/libfuse/libfuse.git + +RUN mkdir -p libfuse/build && cd libfuse/build && meson setup .. && ninja install + +RUN cd libfuse/example && gcc -Wall passthrough.c `pkg-config fuse3 --cflags --libs` -o passthrough + +FROM debian:bullseye-slim@sha256:a165446a88794db4fec31e35e9441433f9552ae048fb1ed26df352d2b537cb96 + +RUN apt update && apt install -y python3 fuse tini supervisor inotify-tools jq && rm -rf /var/cache/apt/* && ln -s /usr/bin/python3 /usr/local/bin/python +COPY inotify-fluid-config.ini /tmp/inotify-fluid-config.ini +RUN cat /tmp/inotify-fluid-config.ini >> /etc/supervisor/supervisord.conf && rm /tmp/inotify-fluid-config.ini + +COPY reconcile_mount_program_settings.py mount-helper.sh inotify.sh mount-passthrough-fuse.sh prestop.sh entrypoint.sh /usr/local/bin/ +RUN chmod u+x /usr/local/bin/mount-helper.sh /usr/local/bin/inotify.sh /usr/local/bin/mount-passthrough-fuse.sh /usr/local/bin/prestop.sh /usr/local/bin/entrypoint.sh + +RUN apt update && apt install -y libfuse3-3 fuse3 +COPY --from=builder libfuse/example/passthrough /usr/local/bin/passthrough + +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] \ No newline at end of file diff --git a/addons/dynamic-mount/base/build.sh b/addons/dynamic-mount/base/build.sh new file mode 100644 index 00000000000..1535506f2c6 --- /dev/null +++ b/addons/dynamic-mount/base/build.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set +x + +docker build . --network=host -f Dockerfile -t fluidcloudnative/dynamic-mount:base + +docker push fluidcloudnative/dynamic-mount:base \ No newline at end of file diff --git a/addons/dynamic-mount/base/entrypoint.sh b/addons/dynamic-mount/base/entrypoint.sh new file mode 100644 index 00000000000..177b6d501be --- /dev/null +++ b/addons/dynamic-mount/base/entrypoint.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +trap "/usr/local/bin/prestop.sh" SIGTERM + +if [[ "$USE_PASSTHROUGH_FUSE" == "True" ]]; then +mkdir -p $MOUNT_POINT +cat << EOF >> /etc/supervisor/supervisord.conf + +[program:passthrough-fuse] +command=/usr/local/bin/mount-passthrough-fuse.sh +redirect_stderr=true +stdout_logfile=/proc/1/fd/1 +stdout_logfile_maxbytes=0 +autorestart=true +startretries=9999 +EOF +fi + +supervisord -n \ No newline at end of file diff --git a/addons/dynamic-mount/base/inotify-fluid-config.ini b/addons/dynamic-mount/base/inotify-fluid-config.ini new file mode 100644 index 00000000000..be839aa9f9e --- /dev/null +++ b/addons/dynamic-mount/base/inotify-fluid-config.ini @@ -0,0 +1,7 @@ +[program:inotify-fluid-config] +command=/usr/local/bin/inotify.sh +redirect_stderr=true +stdout_logfile=/proc/1/fd/1 +stdout_logfile_maxbytes=0 +autorestart=true +startretries=9999 \ No newline at end of file diff --git a/addons/dynamic-mount/base/inotify.sh b/addons/dynamic-mount/base/inotify.sh new file mode 100644 index 00000000000..9e5728ffcfa --- /dev/null +++ b/addons/dynamic-mount/base/inotify.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -xe + +FUSE_CONFIG="/etc/fluid/config" + +python /usr/local/bin/reconcile_mount_program_settings.py +supervisorctl update + +# if fuse-config(/etc/fluid/config/config.json) is modified, reconcile setting files under /etc/supervisor.d and use `supervisorctl update` to start/stop new/old fuse daemon process. +# config.json is mounted by configmap, it is actually a symlink point to actual file, and kubernetes would atomically rename ..data_tmp to ..data, which triggers an inotify moved_to event. +# Please see https://github.com/kubernetes/kubernetes/blob/master/pkg/volume/util/atomic_writer.go#L93-L138 for more information +inotifywait -m -r -e moved_to "${FUSE_CONFIG}" | + while read -r directory event file; do + echo "${directory} ${file} changed (event: ${event})" + # mount_and_umount + python /usr/local/bin/reconcile_mount_program_settings.py + supervisorctl update + done diff --git a/addons/dynamic-mount/base/mount-helper.sh b/addons/dynamic-mount/base/mount-helper.sh new file mode 100644 index 00000000000..a2c4a7c601f --- /dev/null +++ b/addons/dynamic-mount/base/mount-helper.sh @@ -0,0 +1,88 @@ +#!/bin/bash +set -ex + +function help() { + echo "Usage: " + echo " bash mount-helper.sh mount|umount [args...]" + echo "Examples: " + echo " 1. mount filesystem [mount_src] to [mount_target] with options defined in [mount_opt_file]" + echo " bash mount-helper.sh mount [mount_src] [mount_target] [mount_opt_file]" + echo " 2. umount filesystem mounted at [mount_target]" + echo " bash mount-helper.sh umount [mount_target]" +} + +function error_msg() { + help + echo + echo $1 + exit 1 +} + +function clean_up() { + # Ignore any possible error in clean up process + set +e + mount_target=$1 + if [[ -z "$mount_target" ]]; then + return + fi + umount $mount_target + sleep 3 # umount may be asynchronous + rmdir $mount_target +} + +function mount_fn() { + if [[ $# -ne 4 ]]; then + error_msg "Error: mount-helper.sh mount expects 4 arguments, but got $# arguments." + fi + mount_src=$1 + mount_target=$2 + fs_type=$3 + mount_opt_file=$4 + + # NOTES.1: umount $mount_target here to avoid [[ -d $mount_target ]] returning "Transport Endpoint is not connected" error. + # NOTES.2: Use "cat /proc/self/mountinfo" instead of the "mount" command because Alpine has some issue on printing mount info with "mount". + if cat /proc/self/mountinfo | grep " ${mount_target} " > /dev/null; then + echo "found mount point on ${mount_target}, umount it before re-mount." + umount ${mount_target} + fi + + if [[ ! -d "$mount_target" ]]; then + mkdir -p "$mount_target" + fi + + # mount-helper.sh should be wrapped in `tini -s -g` so trap will be triggered + trap "clean_up $mount_target" SIGTERM + /opt/mount.sh $mount_src $mount_target $fs_type $mount_opt_file +} + +function umount_fn() { + if [[ $# -ne 1 ]]; then + error_msg "Error: mount-helper.sh umount expects 1 argument, but got $# arguments." + fi + umount $1 || true +} + +function main() { + if [[ $# -eq 0 ]]; then + error_msg "Error: not enough arguments, require at least 1 argument" + fi + + if [[ $# -gt 0 ]]; then + case $1 in + mount) + shift + mount_fn $@ + ;; + unmount|umount) + shift + umount_fn $@ + ;; + *) + error_msg "Error: unknown option: $1" + ;; + esac + fi +} + +main $@ + diff --git a/addons/dynamic-mount/base/mount-passthrough-fuse.sh b/addons/dynamic-mount/base/mount-passthrough-fuse.sh new file mode 100644 index 00000000000..58258fa3690 --- /dev/null +++ b/addons/dynamic-mount/base/mount-passthrough-fuse.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -ex + +umount $MOUNT_POINT || true +passthrough -o modules=subdir,subdir=/mnt,auto_unmount -f $MOUNT_POINT \ No newline at end of file diff --git a/addons/dynamic-mount/base/prestop.sh b/addons/dynamic-mount/base/prestop.sh new file mode 100644 index 00000000000..26b3f0921f5 --- /dev/null +++ b/addons/dynamic-mount/base/prestop.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -e + +mount_points=$(cat /proc/self/mountinfo | grep " ${MOUNT_POINT}" | awk '{print $5}') + +echo "prestop.sh: umounting mountpoints under ${MOUNT_POINT}" +for mount_point in ${mount_points}; do + echo ">> mount-helper.sh umount ${mount_point}" + mount-helper.sh umount ${mount_point} +done + +# from now on, we clean sub dirs in a best-effort manner. +set +e +echo "prestop.sh: clean sub directories under ${MOUNT_POINT}" +sub_dirs=$(ls "${MOUNT_POINT}/") +for sub_dir in ${sub_dirs}; do + rmdir "${MOUNT_POINT}/${sub_dir}" || echo "WARNING: failed to rmdir ${sub_dir}, maybe filesystem still mounting on it." +done + +exit 0 diff --git a/addons/dynamic-mount/base/reconcile_mount_program_settings.py b/addons/dynamic-mount/base/reconcile_mount_program_settings.py new file mode 100644 index 00000000000..d5e863410a5 --- /dev/null +++ b/addons/dynamic-mount/base/reconcile_mount_program_settings.py @@ -0,0 +1,90 @@ +import json +import glob +import os + +USE_PASSTHROUGH_FUSE = os.environ.get("USE_PASSTHROUGH_FUSE", 'False') == 'True' + +FLUID_RUNTIME_MNT = os.environ.get("MOUNT_POINT") +FLUID_MOUNT_OPT_DIR = "/etc/fluid/mount-opts" +FLUID_CONFIG_FILE = "/etc/fluid/config/config.json" +SUPERVISORD_SETTING_DIR = "/etc/supervisor/conf.d" +SUPERVISORD_SETTING_TEMPLATE = """[program:{name}] +command=tini -s -g -- mount-helper.sh mount {mount_src} {mount_target} {fs_type} {mount_opt_file} +stdout_logfile=/var/logs/fluid/{name}.out +stderr_logfile=/var/logs/fluid/{name}.err +autorestart=true +startretries=9999""" + +def prepare_dirs(): + os.makedirs(SUPERVISORD_SETTING_DIR, exist_ok=True) + os.makedirs("/var/logs/fluid", exist_ok=True) + os.makedirs(FLUID_MOUNT_OPT_DIR, exist_ok=True) + +def write_mount_opts(mount_opts, opt_file): + with open(opt_file, "w") as f: + f.write(json.dumps(mount_opts)) + +def reconcile_supervisord_settings(): + rawStr = "" + with open(FLUID_CONFIG_FILE, "r") as f: + rawStr = f.readlines() + + print(f"{FLUID_CONFIG_FILE}: {rawStr[0]}") # config.json only have one line in json format + + setting_files = glob.glob(os.path.join(SUPERVISORD_SETTING_DIR, "*.conf")) + + # obj["mounts"] is like [{"mountPoint": "s3://mybucket", "name": "mybucket", "path": "/mybucket", "options":{...}}, {"mountPoint": "s3://mybucket2", "name": "mybucket2", "path": "/mybucket2", "options":{...}}] + obj = json.loads(rawStr[0]) + expected_mounts = [mount["name"] for mount in obj["mounts"]] + current_mounts = [os.path.basename(file).removesuffix(".conf") for file in setting_files] + + need_mount = list(set(expected_mounts).difference(set(current_mounts))) + need_unmount = list(set(current_mounts).difference(set(expected_mounts))) + print(f"need mount: {need_mount}, need umount: {need_unmount}") + + for name in need_unmount: + setting_file = os.path.join(SUPERVISORD_SETTING_DIR, f"{name}.conf") + if os.path.isfile(setting_file): + os.remove(setting_file) + print(f"Mount \"{name}\"'s settings has been removed.") + + + access_mode = "ro" + if "ReadWriteMany" in obj["accessModes"]: + access_mode = "rw" + mount_info_dict = {mount["name"]: mount for mount in obj["mounts"]} + for name in need_mount: + if name not in mount_info_dict: + print(f"WARNING: mount \"{name}\" is not found in {FLUID_CONFIG_FILE}.") + continue + mount_info = mount_info_dict[name] + mount_src: str = mount_info["mountPoint"] + fs_type = "unknown" + if len(mount_src.split("://")) == 2: + fs_type = mount_src.split("://")[0] # e.g. mount_src="nfs://xxxx/yyyy" => fs_type=nfs + mount_dir_name = name + if "path" in mount_info: + if mount_info["path"] != "/": + mount_dir_name = mount_info["path"].lstrip("/") + else: + print(f"WARNING: mounting \"{name}\" at \"/\" is not allowed, fall back to mount at \"/{name}\"") + if USE_PASSTHROUGH_FUSE: + mount_target = os.path.join("/mnt", mount_dir_name) + else: + mount_target = os.path.join(FLUID_RUNTIME_MNT, mount_dir_name) + mount_opt_file = os.path.join(FLUID_MOUNT_OPT_DIR, f"{name}.opts") + + mount_opts = mount_info["options"] + mount_opts["name"] = name + mount_opts["access_mode"] = access_mode + write_mount_opts(mount_opts, mount_opt_file) + + setting_file = os.path.join(SUPERVISORD_SETTING_DIR, f"{name}.conf") + with open(setting_file, 'w') as f: + f.write(SUPERVISORD_SETTING_TEMPLATE.format(name=name, mount_src=mount_src, mount_target=mount_target, fs_type=fs_type, mount_opt_file=mount_opt_file)) + + print(f"Mount \"{name}\"'s setting is successfully written to {setting_file}") + +if __name__=="__main__": + prepare_dirs() + reconcile_supervisord_settings() \ No newline at end of file From 909877e3d5233cd2fcabe24fc02cbc7e519bfd25 Mon Sep 17 00:00:00 2001 From: trafalgarzzz Date: Fri, 31 May 2024 15:55:58 +0800 Subject: [PATCH 2/5] Add juicefs examples for fluid dynamic mount feature Signed-off-by: trafalgarzzz --- .../juicefs/docker/Dockerfile.juicefs | 29 +++++++++++++++++++ addons/dynamic-mount/juicefs/docker/mount.sh | 23 +++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 addons/dynamic-mount/juicefs/docker/Dockerfile.juicefs create mode 100644 addons/dynamic-mount/juicefs/docker/mount.sh diff --git a/addons/dynamic-mount/juicefs/docker/Dockerfile.juicefs b/addons/dynamic-mount/juicefs/docker/Dockerfile.juicefs new file mode 100644 index 00000000000..b822756fd98 --- /dev/null +++ b/addons/dynamic-mount/juicefs/docker/Dockerfile.juicefs @@ -0,0 +1,29 @@ +FROM fluidcloudnative/fluid-dynamic-mount-base:v0.4 + +# Install Juicefs +WORKDIR /app + +ARG TARGETARCH +ENV JUICEFS_CLI=/usr/bin/juicefs +ENV JFS_MOUNT_PATH=/usr/local/juicefs/mount/jfsmount + +RUN apt update && apt install -y software-properties-common wget gnupg gnupg2 && bash -c "if [[ '${TARGETARCH}' == amd64 ]]; then wget -O - https://download.gluster.org/pub/gluster/glusterfs/10/rsa.pub | apt-key add - && \ + echo deb [arch=${TARGETARCH}] https://download.gluster.org/pub/gluster/glusterfs/10/LATEST/Debian/buster/${TARGETARCH}/apt buster main > /etc/apt/sources.list.d/gluster.list && \ + apt-get update && apt-get install -y uuid-dev libglusterfs-dev glusterfs-common; fi" + +RUN apt-get update && apt-get install -y librados2 curl fuse procps iputils-ping strace iproute2 net-tools tcpdump lsof librados-dev libcephfs-dev librbd-dev && \ + rm -rf /var/cache/apt/* && \ + bash -c "curl -o ${JUICEFS_CLI} https://juicefs.com/static/juicefs.4.9 && \ + chmod a+x ${JUICEFS_CLI} && mkdir -p /usr/local/juicefs/mount && curl -o ${JFS_MOUNT_PATH} https://juicefs.com/static/Linux/mount.4.9 && chmod a+x ${JFS_MOUNT_PATH};" && \ + chmod +x ${JUICEFS_CLI} && \ + mkdir -p /root/.juicefs && \ + ln -s /usr/local/bin/python /usr/bin/python && \ + mkdir /root/.acl && cp /etc/passwd /root/.acl/passwd && cp /etc/group /root/.acl/group && \ + ln -sf /root/.acl/passwd /etc/passwd && ln -sf /root/.acl/group /etc/group + +RUN /usr/bin/juicefs version + +# Install mount script for dynamic mount +RUN apt install -y jq +COPY mount.sh /opt/mount.sh +RUN chmod u+x /opt/mount.sh \ No newline at end of file diff --git a/addons/dynamic-mount/juicefs/docker/mount.sh b/addons/dynamic-mount/juicefs/docker/mount.sh new file mode 100644 index 00000000000..8db1475ed46 --- /dev/null +++ b/addons/dynamic-mount/juicefs/docker/mount.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -e + +if [[ $# -ne 4 ]]; then + echo "Error: require 3 arguments, but got $# arguments" + exit 1 +fi + +mount_src=$1 # e.g. juicefs://mybucket +mount_target=$2 # e.g. /runtime-mnt/thin/default/thin-demo/thin-fuse/mybucket +fs_type=$3 +mount_opt_file=$4 # e.g. /etc/fluid/mount-opts/mybucket.opts (mount options in json format) + +filesystem_name=${mount_src#juicefs://} +token_file=$(cat ${mount_opt_file} | jq -r '.["token"]') +access_key_file=$(cat ${mount_opt_file} | jq -r '.["access-key"]') +secret_key_file=$(cat ${mount_opt_file} | jq -r '.["secret-key"]') +bucket=$(cat ${mount_opt_file} | jq -r '.["bucket"]') + +juicefs auth $filesystem_name --token `cat $token_file` --access-key `cat $access_key_file` --secret-key `cat $secret_key_file` --bucket "$bucket" + +exec juicefs mount -f $filesystem_name $mount_target \ No newline at end of file From 933827f01c7478d08114a66b4969b381d05ce411 Mon Sep 17 00:00:00 2001 From: trafalgarzzz Date: Thu, 13 Jun 2024 15:38:01 +0800 Subject: [PATCH 3/5] clean up mount point unconditionally Signed-off-by: trafalgarzzz --- addons/dynamic-mount/base/mount-helper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addons/dynamic-mount/base/mount-helper.sh b/addons/dynamic-mount/base/mount-helper.sh index a2c4a7c601f..eabcbeb7c3c 100644 --- a/addons/dynamic-mount/base/mount-helper.sh +++ b/addons/dynamic-mount/base/mount-helper.sh @@ -51,7 +51,7 @@ function mount_fn() { fi # mount-helper.sh should be wrapped in `tini -s -g` so trap will be triggered - trap "clean_up $mount_target" SIGTERM + trap "clean_up $mount_target" SIGTERM EXIT /opt/mount.sh $mount_src $mount_target $fs_type $mount_opt_file } From 25e88ea667b5de2740d960de5edd87e43a31f757 Mon Sep 17 00:00:00 2001 From: trafalgarzzz Date: Thu, 13 Jun 2024 15:54:56 +0800 Subject: [PATCH 4/5] Dump mount point logs to /var/log/fluid Signed-off-by: trafalgarzzz --- .../dynamic-mount/base/reconcile_mount_program_settings.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/addons/dynamic-mount/base/reconcile_mount_program_settings.py b/addons/dynamic-mount/base/reconcile_mount_program_settings.py index d5e863410a5..78f5ec8edd8 100644 --- a/addons/dynamic-mount/base/reconcile_mount_program_settings.py +++ b/addons/dynamic-mount/base/reconcile_mount_program_settings.py @@ -10,14 +10,14 @@ SUPERVISORD_SETTING_DIR = "/etc/supervisor/conf.d" SUPERVISORD_SETTING_TEMPLATE = """[program:{name}] command=tini -s -g -- mount-helper.sh mount {mount_src} {mount_target} {fs_type} {mount_opt_file} -stdout_logfile=/var/logs/fluid/{name}.out -stderr_logfile=/var/logs/fluid/{name}.err +stdout_logfile=/var/log/fluid/{name}.out +stderr_logfile=/var/log/fluid/{name}.err autorestart=true startretries=9999""" def prepare_dirs(): os.makedirs(SUPERVISORD_SETTING_DIR, exist_ok=True) - os.makedirs("/var/logs/fluid", exist_ok=True) + os.makedirs("/var/log/fluid", exist_ok=True) os.makedirs(FLUID_MOUNT_OPT_DIR, exist_ok=True) def write_mount_opts(mount_opts, opt_file): From fc08e30d98c5eb804dee27620d3629452d6ee3da Mon Sep 17 00:00:00 2001 From: trafalgarzzz Date: Thu, 13 Jun 2024 16:08:10 +0800 Subject: [PATCH 5/5] Anchor base image for dynamic mount example Signed-off-by: trafalgarzzz --- addons/dynamic-mount/base/Dockerfile | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/addons/dynamic-mount/base/Dockerfile b/addons/dynamic-mount/base/Dockerfile index b4b2ed70d8e..e43a0e64ee9 100644 --- a/addons/dynamic-mount/base/Dockerfile +++ b/addons/dynamic-mount/base/Dockerfile @@ -1,11 +1,7 @@ -FROM debian:bullseye as builder - -RUN sed -i 's/deb.debian.org/mirrors.aliyun.com/g' /etc/apt/sources.list +FROM debian:bullseye@sha256:a165446a88794db4fec31e35e9441433f9552ae048fb1ed26df352d2b537cb96 as builder RUN apt update && apt install -y build-essential libfuse3-dev pkg-config git python3-pip -RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple - RUN pip install meson ninja RUN git clone https://github.com/libfuse/libfuse.git