diff --git a/build/gather-node-deps.sh b/build/gather-node-deps.sh new file mode 100755 index 000000000..7bd00f133 --- /dev/null +++ b/build/gather-node-deps.sh @@ -0,0 +1,38 @@ +#!/bin/bash +set -e +mkdir /staging-node +DEPS=( + /etc/mke2fs.conf + /usr/bin/{mount,umount,lspci,mkdir,chmod,grep,tail,nsenter} + /usr/sbin/{mount.*,umount.*,fsck,fsck.*,mkfs,mkfs.*,*2fs,xfs_*,sfdisk,losetup,blockdev} +) +MTIME=0 +gather_dep() { + if [ -e "/base$1" ]; then + echo "$1 already exist in base" + return 0 + fi + local source target t + # resolve all but last component of symlink + source=$(realpath "$(dirname "$1")")/$(basename "$1") + [ -e "/staging-node$source" ] && return 0 + if [ -h "$source" ]; then + target=$(realpath "$source") + echo "gathering link $source => $target" + gather_dep "$target" + fi + echo "gathering dep $source" + t=$(stat -c '%Y' "$source") + [ "$t" -gt "$MTIME" ] && MTIME=$t + cp -dp --parents "$source" /staging-node +} +for f in "${DEPS[@]}"; do + gather_dep "$f" +done + +mapfile -t LIBS < <(ldd /staging-node/usr/{bin,sbin}/* 2>/dev/null | grep -Po '(?<= => )[^ ]+' | sort | uniq) +for f in "${LIBS[@]}"; do + gather_dep "$f" +done +echo "latest mtime is $(date --date "@$MTIME" --iso-8601=seconds)" +find /staging-node -type d -exec touch --date="@$MTIME" {} + diff --git a/build/lib/init.sh b/build/lib/init.sh index 174fddaea..7111b436e 100755 --- a/build/lib/init.sh +++ b/build/lib/init.sh @@ -1,4 +1,5 @@ #!/bin/sh +# shellcheck shell=busybox # skip all the setup if running in provisioner mode if [ "$SERVICE_TYPE" = "provisioner" ]; then @@ -16,30 +17,18 @@ run_nas="false" mkdir -p /var/log/alicloud/ mkdir -p /host/etc/kubernetes/volumes/disk/uuid -HOST_CMD="/usr/bin/nsenter --mount=/proc/1/ns/mnt" +host_cmd() { nsenter --mount=/proc/1/ns/mnt "$@"; } HOST_ROOT="/proc/1/root" -host_os="centos" - OS_RELEASE=/etc/os-release if ! [ -f $HOST_ROOT$OS_RELEASE ]; then OS_RELEASE=/usr/lib/os-release fi -if [ -f $HOST_ROOT$OS_RELEASE ]; then - OS_RELEASE_ID=$(grep -oP '(?<=^ID=).+' $HOST_ROOT$OS_RELEASE | tr -d '"') - if [ -n "$OS_RELEASE_ID" ]; then - host_os="$OS_RELEASE_ID" - - if [[ $OS_RELEASE_ID = alinux ]]; then - OS_RELEASE_VERSION_ID=$(grep -oP '(?<=^VERSION_ID=).+' $HOST_ROOT$OS_RELEASE | tr -d '"') - host_os="alinux$(echo $OS_RELEASE_VERSION_ID | cut -d'.' -f1)" - fi - fi -fi -echo "detected host os: $host_os" +OS_RELEASE_ID=$(. $HOST_ROOT$OS_RELEASE; echo "$ID") +OS_RELEASE_VERSION_ID=$(. $HOST_ROOT$OS_RELEASE; echo "$VERSION_ID") ARCH=$(uname -m) -echo "detected host arch: $ARCH" +echo "detected host OS: $OS_RELEASE_ID, version: $OS_RELEASE_VERSION_ID, arch: $ARCH" cleanup_old_staging_path() ( echo cleaning up old volume staging path. # kubelet will mount the new path at startup. @@ -60,7 +49,7 @@ if [ -d "$OLD_STAGING_PATH" ]; then fi ## check which plugin is running -for item in $@; +for item in "$@"; do if [ "$item" = "--driver=ossplugin.csi.alibabacloud.com" ]; then echo "Running oss plugin...." @@ -77,11 +66,9 @@ do run_nas="true" mkdir -p "$KUBELET_ROOT_DIR/csi-plugins/nasplugin.csi.alibabacloud.com" rm -rf "$KUBELET_ROOT_DIR/plugins/nasplugin.csi.alibabacloud.com/csi.sock" - elif [[ $item==*--driver=* ]]; then - tmp=${item} - driver_types=${tmp#*--driver=} - driver_type_array=(${driver_types//,/ }) - for driver_type in ${driver_type_array[@]}; + elif [[ "$item" = *--driver=* ]]; then + IFS=, + for driver_type in ${item#*--driver=}; do if [ "$driver_type" = "oss" ]; then echo "Running oss plugin...." @@ -103,6 +90,7 @@ do run_pov="true" fi done + unset IFS fi done @@ -112,12 +100,12 @@ if [ "$SKIP_UPDATEDB_CONFIG" != "true" ]; then if [ -f /host/etc/cron.daily/mlocate ]; then if [ -f /host/etc/updatedb.conf ]; then sed -i 's/PRUNE_BIND_MOUNTS.*$/PRUNE_BIND_MOUNTS = \"yes\"/g' /host/etc/updatedb.conf - if ! grep "^PRUNEFS" /host/etc/updatedb.conf | grep -q --fixed-strings "fuse.ossfs"; then + if ! grep "^PRUNEFS" /host/etc/updatedb.conf | grep -q -F "fuse.ossfs"; then PRUNEFS="fuse.ossfs" echo "add PRUNEFS: $PRUNEFS to /etc/updatedb.conf" sed -i "s|PRUNEFS\s*=\s*\"|&${PRUNEFS//|/\\|} |" /host/etc/updatedb.conf fi - if ! grep "^PRUNEPATHS" /host/etc/updatedb.conf | grep -q --fixed-strings "$KUBELET_ROOT_DIR"; then + if ! grep "^PRUNEPATHS" /host/etc/updatedb.conf | grep -q -F "$KUBELET_ROOT_DIR"; then PRUNEPATHS="$KUBELET_ROOT_DIR" if [ "$KUBELET_ROOT_DIR" = "/var/lib/kubelet" ]; then PRUNEPATHS="$PRUNEPATHS /var/lib/container/kubelet" @@ -134,15 +122,15 @@ if [ "$DISABLE_CSIPLUGIN_CONNECTOR" != "true" ] && [ "$run_nas" = "true" ]; then ## install/update csi connector updateConnector="true" systemdDir="/host/usr/lib/systemd/system" - if [[ ${host_os} == "lifsea" ]]; then + if [ "$OS_RELEASE_ID" = "lifsea" ]; then systemdDir="/host/etc/systemd/system" fi if [ ! -f "/host/etc/csi-tool/csiplugin-connector" ]; then mkdir -p /host/etc/csi-tool/ echo "mkdir /etc/csi-tool/ directory..." else - oldmd5=`md5sum /host/etc/csi-tool/csiplugin-connector | awk '{print $1}'` - newmd5=`md5sum /csi/csiplugin-connector | awk '{print $1}'` + oldmd5=$(md5sum /host/etc/csi-tool/csiplugin-connector | awk '{print $1}') + newmd5=$(md5sum /csi/csiplugin-connector | awk '{print $1}') if [ "$oldmd5" = "$newmd5" ]; then updateConnector="false" else @@ -163,14 +151,14 @@ if [ "$DISABLE_CSIPLUGIN_CONNECTOR" != "true" ] && [ "$run_nas" = "true" ]; then # install/update csiplugin connector service updateConnectorService="true" - if [[ ! -z "${PLUGINS_SOCKETS}" ]];then + if [ -n "${PLUGINS_SOCKETS}" ];then sed -i 's/Restart=always/Restart=on-failure/g' /csi/csiplugin-connector.service sed -i '/^\[Service\]/a Environment=\"WATCHDOG_SOCKETS_PATH='"${PLUGINS_SOCKETS}"'\"' /csi/csiplugin-connector.service fi if [ -f "$systemdDir/csiplugin-connector.service" ];then echo "Check csiplugin-connector.service...." - oldmd5=`md5sum $systemdDir/csiplugin-connector.service | awk '{print $1}'` - newmd5=`md5sum /csi/csiplugin-connector.service | awk '{print $1}'` + oldmd5=$(md5sum $systemdDir/csiplugin-connector.service | awk '{print $1}') + newmd5=$(md5sum /csi/csiplugin-connector.service | awk '{print $1}') if [ "$oldmd5" = "$newmd5" ]; then updateConnectorService="false" else @@ -182,10 +170,9 @@ if [ "$DISABLE_CSIPLUGIN_CONNECTOR" != "true" ] && [ "$run_nas" = "true" ]; then echo "Install csiplugin connector service...." cp /csi/csiplugin-connector.service $systemdDir/csiplugin-connector.service echo "Starting systemctl daemon-reload." - for((i=1;i<=10;i++)); + for i in $(seq 10); do - ${HOST_CMD} systemctl daemon-reload - if [ $? -eq 0 ]; then + if host_cmd systemctl daemon-reload; then break else echo "Starting retry again systemctl daemon-reload.retry count:$i" @@ -196,10 +183,9 @@ if [ "$DISABLE_CSIPLUGIN_CONNECTOR" != "true" ] && [ "$run_nas" = "true" ]; then rm -rf /var/log/alicloud/connector.pid echo "Starting systemctl enable csiplugin-connector.service." - for((i=1;i<=5;i++)); + for i in $(seq 5); do - ${HOST_CMD} systemctl enable csiplugin-connector.service - if [ $? -eq 0 ]; then + if host_cmd systemctl enable csiplugin-connector.service; then break else echo "Starting retry again systemctl enable csiplugin-connector.service.retry count:$i" @@ -208,10 +194,9 @@ if [ "$DISABLE_CSIPLUGIN_CONNECTOR" != "true" ] && [ "$run_nas" = "true" ]; then done echo "Starting systemctl restart csiplugin-connector.service." - for((i=1;i<=5;i++)); + for i in $(seq 5); do - ${HOST_CMD} systemctl restart csiplugin-connector.service - if [ $? -eq 0 ]; then + if host_cmd systemctl restart csiplugin-connector.service; then break else echo "Starting retry again systemctl restart csiplugin-connector.service.retry count:$i" @@ -222,49 +207,60 @@ fi echo "Start checking if the host system packages needs to be installed" -if $HOST_CMD dnf --version; then - echo "dnf is available" - PKG_CMD=(dnf install -y) - HAS_DNF=true - HAS_YUM=true -elif $HOST_CMD yum --version; then - echo "yum is available" +dnf_install() { + host_cmd dnf install -y "$@" +} + +yum_install() { # yum install is not idempotent, so we need to check if the package is already installed. # yum also fails if the already installed version is newer, so we need to downgrade. # fail if both install and downgrade fail - PKG_CMD=(sh -c 'if rpm --query $(rpm --query --package $1); then - echo $1 already installed; + # shellcheck disable=SC2016 # the expansion should happen on host, not in container. + host_cmd sh -c 'if rpm --query $(rpm --query --package "$1"); then + echo "$1 already installed"; else - yum install -y $1 || yum downgrade -y $1; - fi' install_pkg) - HAS_YUM=true -elif $HOST_CMD apt-get --version; then + yum install -y "$1" || yum downgrade -y "$1"; + fi' install_pkg "$1" +} + +apt_install() { + host_cmd apt-get install -y --allow-downgrades "$@" +} + +if host_cmd dnf --version; then + echo "dnf is available" + PKG_MGR=dnf + SUPPORT_RPM=true +elif host_cmd yum --version; then + echo "yum is available" + PKG_MGR=yum + SUPPORT_RPM=true +elif host_cmd apt-get --version; then echo "apt-get is available" - PKG_CMD=(apt-get install -y --allow-downgrades) - HAS_APT=true + PKG_MGR=apt fi # idempotent package installation # downgrade if the already installed version is newer than the one we want to install -function install_package() { - cp /root/$1 /host/etc/csi-tool/ - if [ -z $PKG_CMD ]; then +install_package() { + cp "/root/$1" /host/etc/csi-tool/ + if [ -z "$PKG_MGR" ]; then echo "no package manager found" return 1 fi - ${HOST_CMD} "${PKG_CMD[@]}" /etc/csi-tool/$1 || { + "${PKG_MGR}_install" "/etc/csi-tool/$1" || { echo "failed to install $1" return 1 } - rm -f /host/etc/csi-tool/$1 + rm -f "/host/etc/csi-tool/$1" } -if ([ "$DISK_BDF_ENABLE" = "true" ] && [ "$run_disk" = "true" ]) || [ "$run_pov" = "true" ]; then +if { [ "$DISK_BDF_ENABLE" = "true" ] && [ "$run_disk" = "true" ]; } || [ "$run_pov" = "true" ]; then isbdf="false" - for i in $(${HOST_CMD} lspci -D | grep "storage controller" | grep "1ded\|Alibaba" | awk '{print $1}' | sed -n '/0$/p'); + for i in $(host_cmd lspci -D | grep "storage controller" | grep "1ded\|Alibaba" | awk '{print $1}' | sed -n '/0$/p'); do - out=`${HOST_CMD} lspci -s $i -v`; - if [[ $out == *"Single Root I/O Virtualization"* ]]; then + out=$(host_cmd lspci -s "$i" -v); + if [[ "$out" = *"Single Root I/O Virtualization"* ]]; then isbdf="true" break fi @@ -273,15 +269,14 @@ if ([ "$DISK_BDF_ENABLE" = "true" ] && [ "$run_disk" = "true" ]) || [ "$run_pov" echo "isbdf node: $isbdf" if [ $isbdf = "true" ]; then echo "start install vfhp" - ${HOST_CMD} yum install -y "http://yum.tbsite.net/taobao/7/$ARCH/current/iohub-vfhp-helper/iohub-vfhp-helper-0.1.9-20231121192230.$ARCH.rpm" - if [ $? -ne 0 ]; then - ${HOST_CMD} yum install -y "https://iohub-vfhp-helper.oss-rg-china-mainland.aliyuncs.com/iohub-vfhp-helper-0.1.9-20231121192230.$ARCH.rpm" + if ! host_cmd yum install -y "http://yum.tbsite.net/taobao/7/$ARCH/current/iohub-vfhp-helper/iohub-vfhp-helper-0.1.9-20231121192230.$ARCH.rpm"; then + host_cmd yum install -y "https://iohub-vfhp-helper.oss-rg-china-mainland.aliyuncs.com/iohub-vfhp-helper-0.1.9-20231121192230.$ARCH.rpm" fi # takes 10s - output=`${HOST_CMD} iohub-vfhp-helper -s` - if [[ $output == *"backend support auto vf hotplug."* ]]; then + output=$(host_cmd iohub-vfhp-helper -s) + if [[ "$output" == *"backend support auto vf hotplug."* ]]; then echo "backend support auto vf hotplugin" - ${HOST_CMD} sudo service iohub-vfhp-helper start + host_cmd sudo service iohub-vfhp-helper start else echo "backend not support auto vf hotplugin" fi @@ -309,20 +304,20 @@ if [ "$ARCH" = "x86_64" ] && [ "$run_nas" = "true" ]; then if [ $install_utils = "true" ]; then # cpfs-nas nas-rich-client common rpm echo "installing aliyun-alinas-utils" - if [ "$HAS_YUM" = "true" ]; then + if [ "$SUPPORT_RPM" = "true" ]; then PKG=aliyun-alinas-utils-1.1-8.20240527201444.2012cc.al7.noarch.rpm - elif [ "$HAS_APT" = "true" ]; then + elif [ "$PKG_MGR" = "apt" ]; then PKG=aliyun-alinas-utils-1.1-8.deb else echo "WARN: no supported package manager found, skip install aliyun-alinas-utils" install_utils="false" fi if [ $install_utils = "true" ]; then - install_package $PKG || exit $? + install_package "$PKG" || exit $? fi fi - if [ $host_os != "alinux2" ] && [ $install_efc = "true" ]; then + if { [ "$OS_RELEASE_ID" != "alinux" ] || [[ "$OS_RELEASE_VERSION_ID" != 2.* ]]; } && [ $install_efc = "true" ]; then echo "WARN: skip install efc because host os is not alinux2" install_efc="false" fi @@ -330,14 +325,14 @@ if [ "$ARCH" = "x86_64" ] && [ "$run_nas" = "true" ]; then # nas-rich-client rpm echo "installing alinas-efc" cp /root/alinas-efc-1.2-3.x86_64.rpm /host/etc/csi-tool/ - ${HOST_CMD} yum install -y /etc/csi-tool/alinas-efc-1.2-3.x86_64.rpm + host_cmd yum install -y /etc/csi-tool/alinas-efc-1.2-3.x86_64.rpm echo "checking alinas-efc-1.2-3.x86_64 installed" - ${HOST_CMD} rpm -q alinas-efc-1.2-3.x86_64 || exit 1 + host_cmd rpm -q alinas-efc-1.2-3.x86_64 || exit 1 echo "starting aliyun-alinas-mount-watchdog" - ${HOST_CMD} systemctl start aliyun-alinas-mount-watchdog || exit 1 + host_cmd systemctl start aliyun-alinas-mount-watchdog || exit 1 fi fi # place it here to remove leftover from previous version -rm -rf /host/etc/csi-tool/*.{rpm,deb} +rm -rf /host/etc/csi-tool/*.rpm /host/etc/csi-tool/*.deb diff --git a/build/multi/Dockerfile.multi b/build/multi/Dockerfile.multi index db24418f8..cb5f7eb10 100644 --- a/build/multi/Dockerfile.multi +++ b/build/multi/Dockerfile.multi @@ -16,43 +16,78 @@ RUN --mount=type=bind,target=. \ -o /out/plugin.csi.alibabacloud.com && \ go build -o /out/csiplugin-connector ./build/lib/csiplugin-connector.go - -FROM --platform=$BUILDPLATFORM registry.eu-west-1.aliyuncs.com/acs/alinux:3-update as download -ARG TARGETARCH -WORKDIR /downloads -RUN CURL="curl --remote-time --remote-name --fail" && \ - if [[ $TARGETARCH == "amd64" ]]; then \ - $CURL -H "If-Match: D7AE0A7E48FBB6CF7BDB33C770B5EE0C" https://aliyun-alinas-eac.oss-cn-beijing.aliyuncs.com/alinas-efc-1.2-3.x86_64.rpm && \ - $CURL -H "If-Match: 766DCD5965BF89DE927DC3975728F676" https://aliyun-encryption.oss-cn-beijing.aliyuncs.com/aliyun-alinas-utils-1.1-8.20240527201444.2012cc.al7.noarch.rpm && \ - $CURL -H "If-Match: 259C07CF1C83C71DD78B705BC2E3FAE2" https://aliyun-encryption.oss-cn-beijing.aliyuncs.com/aliyun-alinas-utils-1.1-8.deb; \ - fi - -FROM registry.eu-west-1.aliyuncs.com/acs/alinux:3-update as base +FROM gcr.io/distroless/base-debian12:latest as distroless-base LABEL maintainers="Alibaba Cloud Authors" description="Alibaba Cloud CSI Plugin" -ARG TARGETARCH -RUN --mount=type=cache,target=/var/cache/dnf,sharing=locked,id=dnf-cache-$TARGETARCH \ - dnf install -y ca-certificates file tzdata nfs-utils xfsprogs e4fsprogs pciutils iputils strace cloud-utils-growpart gdisk nc telnet tar cpio && \ - ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && echo 'Asia/Shanghai' >/etc/timezone +FROM distroless-base as csi-controller +COPY --link --from=build /out/plugin.csi.alibabacloud.com /usr/bin/plugin.csi.alibabacloud.com +ENTRYPOINT ["/usr/bin/plugin.csi.alibabacloud.com"] + +FROM debian:bookworm-20240612-slim as debian -FROM base as build-util-linux ARG TARGETARCH -RUN --mount=type=cache,target=/var/cache/dnf,sharing=locked,id=dnf-cache-$TARGETARCH \ - dnf install -y tar xz make gcc diffutils +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-cache-$TARGETARCH \ + --mount=type=cache,target=/var/lib/apt,sharing=locked,id=apt-lib-$TARGETARCH \ + rm -f /etc/apt/apt.conf.d/docker-clean && \ + sed -i s/deb.debian.org/mirrors.aliyun.com/ /etc/apt/sources.list.d/debian.sources && \ + apt-get update && \ + apt-get install -y nfs-common e2fsprogs xfsprogs pciutils fdisk + +RUN --mount=type=bind,from=distroless-base,target=/base \ + --mount=type=bind,source=build/gather-node-deps.sh,target=/deps.sh \ + /deps.sh + +FROM --platform=$BUILDPLATFORM debian:bookworm-20240612-slim as build-0 +ARG BUILDARCH +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-cache-$BUILDARCH \ + --mount=type=cache,target=/var/lib/apt,sharing=locked,id=apt-lib-$BUILDARCH \ + rm -f /etc/apt/apt.conf.d/docker-clean && \ + sed -i s/deb.debian.org/mirrors.aliyun.com/ /etc/apt/sources.list.d/debian.sources && \ + apt-get update && \ + apt-get install -y tar xz-utils make diffutils + +FROM build-0 as build-util-linux-amd64 +ENV HOST=x86_64-linux-gnu + +FROM build-0 as build-util-linux-arm64 +ENV HOST=aarch64-linux-gnu + +FROM build-util-linux-$TARGETARCH as build-util-linux +ARG BUILDARCH +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-cache-$BUILDARCH \ + --mount=type=cache,target=/var/lib/apt,sharing=locked,id=apt-lib-$BUILDARCH <