Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

build: rewrite dockerfile to distroless #1098

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

build: rewrite dockerfile to distroless #1098

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
89f9e9c
init: fix for busybox
huww98 Jun 27, 2024
bd5c158
build: rewrite dockerfile to distroless
huww98 Jun 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions build/gather-node-deps.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
#!/bin/bash
set -e
mkdir /staging-node
DEPS=(
/etc/mke2fs.conf
/usr/bin/{mount,umount,lspci,mkdir,chmod,grep,tail,nsenter}
/usr/sbin/{mount.*,umount.*,fsck,fsck.*,mkfs,mkfs.*,*2fs,xfs_*,sfdisk,losetup,blockdev}
)
MTIME=0
gather_dep() {
if [ -e "/base$1" ]; then
echo "$1 already exist in base"
return 0
fi
local source target t
# resolve all but last component of symlink
source=$(realpath "$(dirname "$1")")/$(basename "$1")
[ -e "/staging-node$source" ] && return 0
if [ -h "$source" ]; then
target=$(realpath "$source")
echo "gathering link $source => $target"
gather_dep "$target"
fi
echo "gathering dep $source"
t=$(stat -c '%Y' "$source")
[ "$t" -gt "$MTIME" ] && MTIME=$t
cp -dp --parents "$source" /staging-node
}
for f in "${DEPS[@]}"; do
gather_dep "$f"
done

mapfile -t LIBS < <(ldd /staging-node/usr/{bin,sbin}/* 2>/dev/null | grep -Po '(?<= => )[^ ]+' | sort | uniq)
for f in "${LIBS[@]}"; do
gather_dep "$f"
done
echo "latest mtime is $(date --date "@$MTIME" --iso-8601=seconds)"
find /staging-node -type d -exec touch --date="@$MTIME" {} +
149 changes: 72 additions & 77 deletions build/lib/init.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#!/bin/sh
# shellcheck shell=busybox

# skip all the setup if running in provisioner mode
if [ "$SERVICE_TYPE" = "provisioner" ]; then
Expand All @@ -16,30 +17,18 @@ run_nas="false"
mkdir -p /var/log/alicloud/
mkdir -p /host/etc/kubernetes/volumes/disk/uuid

HOST_CMD="/usr/bin/nsenter --mount=/proc/1/ns/mnt"
host_cmd() { nsenter --mount=/proc/1/ns/mnt "$@"; }
HOST_ROOT="/proc/1/root"

host_os="centos"

OS_RELEASE=/etc/os-release
if ! [ -f $HOST_ROOT$OS_RELEASE ]; then
OS_RELEASE=/usr/lib/os-release
fi
if [ -f $HOST_ROOT$OS_RELEASE ]; then
OS_RELEASE_ID=$(grep -oP '(?<=^ID=).+' $HOST_ROOT$OS_RELEASE | tr -d '"')
if [ -n "$OS_RELEASE_ID" ]; then
host_os="$OS_RELEASE_ID"

if [[ $OS_RELEASE_ID = alinux ]]; then
OS_RELEASE_VERSION_ID=$(grep -oP '(?<=^VERSION_ID=).+' $HOST_ROOT$OS_RELEASE | tr -d '"')
host_os="alinux$(echo $OS_RELEASE_VERSION_ID | cut -d'.' -f1)"
fi
fi
fi
echo "detected host os: $host_os"

OS_RELEASE_ID=$(. $HOST_ROOT$OS_RELEASE; echo "$ID")
OS_RELEASE_VERSION_ID=$(. $HOST_ROOT$OS_RELEASE; echo "$VERSION_ID")
ARCH=$(uname -m)
echo "detected host arch: $ARCH"
echo "detected host OS: $OS_RELEASE_ID, version: $OS_RELEASE_VERSION_ID, arch: $ARCH"

cleanup_old_staging_path() (
echo cleaning up old volume staging path. # kubelet will mount the new path at startup.
Expand All @@ -60,7 +49,7 @@ if [ -d "$OLD_STAGING_PATH" ]; then
fi

## check which plugin is running
for item in $@;
for item in "$@";
do
if [ "$item" = "--driver=ossplugin.csi.alibabacloud.com" ]; then
echo "Running oss plugin...."
Expand All @@ -77,11 +66,9 @@ do
run_nas="true"
mkdir -p "$KUBELET_ROOT_DIR/csi-plugins/nasplugin.csi.alibabacloud.com"
rm -rf "$KUBELET_ROOT_DIR/plugins/nasplugin.csi.alibabacloud.com/csi.sock"
elif [[ $item==*--driver=* ]]; then
tmp=${item}
driver_types=${tmp#*--driver=}
driver_type_array=(${driver_types//,/ })
for driver_type in ${driver_type_array[@]};
elif [[ "$item" = *--driver=* ]]; then
IFS=,
for driver_type in ${item#*--driver=};
do
if [ "$driver_type" = "oss" ]; then
echo "Running oss plugin...."
Expand All @@ -103,6 +90,7 @@ do
run_pov="true"
fi
done
unset IFS
fi
done

Expand All @@ -112,12 +100,12 @@ if [ "$SKIP_UPDATEDB_CONFIG" != "true" ]; then
if [ -f /host/etc/cron.daily/mlocate ]; then
if [ -f /host/etc/updatedb.conf ]; then
sed -i 's/PRUNE_BIND_MOUNTS.*$/PRUNE_BIND_MOUNTS = \"yes\"/g' /host/etc/updatedb.conf
if ! grep "^PRUNEFS" /host/etc/updatedb.conf | grep -q --fixed-strings "fuse.ossfs"; then
if ! grep "^PRUNEFS" /host/etc/updatedb.conf | grep -q -F "fuse.ossfs"; then
PRUNEFS="fuse.ossfs"
echo "add PRUNEFS: $PRUNEFS to /etc/updatedb.conf"
sed -i "s|PRUNEFS\s*=\s*\"|&${PRUNEFS//|/\\|} |" /host/etc/updatedb.conf
fi
if ! grep "^PRUNEPATHS" /host/etc/updatedb.conf | grep -q --fixed-strings "$KUBELET_ROOT_DIR"; then
if ! grep "^PRUNEPATHS" /host/etc/updatedb.conf | grep -q -F "$KUBELET_ROOT_DIR"; then
PRUNEPATHS="$KUBELET_ROOT_DIR"
if [ "$KUBELET_ROOT_DIR" = "/var/lib/kubelet" ]; then
PRUNEPATHS="$PRUNEPATHS /var/lib/container/kubelet"
Expand All @@ -134,15 +122,15 @@ if [ "$DISABLE_CSIPLUGIN_CONNECTOR" != "true" ] && [ "$run_nas" = "true" ]; then
## install/update csi connector
updateConnector="true"
systemdDir="/host/usr/lib/systemd/system"
if [[ ${host_os} == "lifsea" ]]; then
if [ "$OS_RELEASE_ID" = "lifsea" ]; then
systemdDir="/host/etc/systemd/system"
fi
if [ ! -f "/host/etc/csi-tool/csiplugin-connector" ]; then
mkdir -p /host/etc/csi-tool/
echo "mkdir /etc/csi-tool/ directory..."
else
oldmd5=`md5sum /host/etc/csi-tool/csiplugin-connector | awk '{print $1}'`
newmd5=`md5sum /csi/csiplugin-connector | awk '{print $1}'`
oldmd5=$(md5sum /host/etc/csi-tool/csiplugin-connector | awk '{print $1}')
newmd5=$(md5sum /csi/csiplugin-connector | awk '{print $1}')
if [ "$oldmd5" = "$newmd5" ]; then
updateConnector="false"
else
Expand All @@ -163,14 +151,14 @@ if [ "$DISABLE_CSIPLUGIN_CONNECTOR" != "true" ] && [ "$run_nas" = "true" ]; then

# install/update csiplugin connector service
updateConnectorService="true"
if [[ ! -z "${PLUGINS_SOCKETS}" ]];then
if [ -n "${PLUGINS_SOCKETS}" ];then
sed -i 's/Restart=always/Restart=on-failure/g' /csi/csiplugin-connector.service
sed -i '/^\[Service\]/a Environment=\"WATCHDOG_SOCKETS_PATH='"${PLUGINS_SOCKETS}"'\"' /csi/csiplugin-connector.service
fi
if [ -f "$systemdDir/csiplugin-connector.service" ];then
echo "Check csiplugin-connector.service...."
oldmd5=`md5sum $systemdDir/csiplugin-connector.service | awk '{print $1}'`
newmd5=`md5sum /csi/csiplugin-connector.service | awk '{print $1}'`
oldmd5=$(md5sum $systemdDir/csiplugin-connector.service | awk '{print $1}')
newmd5=$(md5sum /csi/csiplugin-connector.service | awk '{print $1}')
if [ "$oldmd5" = "$newmd5" ]; then
updateConnectorService="false"
else
Expand All @@ -182,10 +170,9 @@ if [ "$DISABLE_CSIPLUGIN_CONNECTOR" != "true" ] && [ "$run_nas" = "true" ]; then
echo "Install csiplugin connector service...."
cp /csi/csiplugin-connector.service $systemdDir/csiplugin-connector.service
echo "Starting systemctl daemon-reload."
for((i=1;i<=10;i++));
for i in $(seq 10);
do
${HOST_CMD} systemctl daemon-reload
if [ $? -eq 0 ]; then
if host_cmd systemctl daemon-reload; then
break
else
echo "Starting retry again systemctl daemon-reload.retry count:$i"
Expand All @@ -196,10 +183,9 @@ if [ "$DISABLE_CSIPLUGIN_CONNECTOR" != "true" ] && [ "$run_nas" = "true" ]; then

rm -rf /var/log/alicloud/connector.pid
echo "Starting systemctl enable csiplugin-connector.service."
for((i=1;i<=5;i++));
for i in $(seq 5);
do
${HOST_CMD} systemctl enable csiplugin-connector.service
if [ $? -eq 0 ]; then
if host_cmd systemctl enable csiplugin-connector.service; then
break
else
echo "Starting retry again systemctl enable csiplugin-connector.service.retry count:$i"
Expand All @@ -208,10 +194,9 @@ if [ "$DISABLE_CSIPLUGIN_CONNECTOR" != "true" ] && [ "$run_nas" = "true" ]; then
done

echo "Starting systemctl restart csiplugin-connector.service."
for((i=1;i<=5;i++));
for i in $(seq 5);
do
${HOST_CMD} systemctl restart csiplugin-connector.service
if [ $? -eq 0 ]; then
if host_cmd systemctl restart csiplugin-connector.service; then
break
else
echo "Starting retry again systemctl restart csiplugin-connector.service.retry count:$i"
Expand All @@ -222,49 +207,60 @@ fi

echo "Start checking if the host system packages needs to be installed"

if $HOST_CMD dnf --version; then
echo "dnf is available"
PKG_CMD=(dnf install -y)
HAS_DNF=true
HAS_YUM=true
elif $HOST_CMD yum --version; then
echo "yum is available"
dnf_install() {
host_cmd dnf install -y "$@"
}

yum_install() {
# yum install is not idempotent, so we need to check if the package is already installed.
# yum also fails if the already installed version is newer, so we need to downgrade.
# fail if both install and downgrade fail
PKG_CMD=(sh -c 'if rpm --query $(rpm --query --package $1); then
echo $1 already installed;
# shellcheck disable=SC2016 # the expansion should happen on host, not in container.
host_cmd sh -c 'if rpm --query $(rpm --query --package "$1"); then
echo "$1 already installed";
else
yum install -y $1 || yum downgrade -y $1;
fi' install_pkg)
HAS_YUM=true
elif $HOST_CMD apt-get --version; then
yum install -y "$1" || yum downgrade -y "$1";
fi' install_pkg "$1"
}

apt_install() {
host_cmd apt-get install -y --allow-downgrades "$@"
}

if host_cmd dnf --version; then
echo "dnf is available"
PKG_MGR=dnf
SUPPORT_RPM=true
elif host_cmd yum --version; then
echo "yum is available"
PKG_MGR=yum
SUPPORT_RPM=true
elif host_cmd apt-get --version; then
echo "apt-get is available"
PKG_CMD=(apt-get install -y --allow-downgrades)
HAS_APT=true
PKG_MGR=apt
fi

# idempotent package installation
# downgrade if the already installed version is newer than the one we want to install
function install_package() {
cp /root/$1 /host/etc/csi-tool/
if [ -z $PKG_CMD ]; then
install_package() {
cp "/root/$1" /host/etc/csi-tool/
if [ -z "$PKG_MGR" ]; then
echo "no package manager found"
return 1
fi
${HOST_CMD} "${PKG_CMD[@]}" /etc/csi-tool/$1 || {
"${PKG_MGR}_install" "/etc/csi-tool/$1" || {
echo "failed to install $1"
return 1
}
rm -f /host/etc/csi-tool/$1
rm -f "/host/etc/csi-tool/$1"
}

if ([ "$DISK_BDF_ENABLE" = "true" ] && [ "$run_disk" = "true" ]) || [ "$run_pov" = "true" ]; then
if { [ "$DISK_BDF_ENABLE" = "true" ] && [ "$run_disk" = "true" ]; } || [ "$run_pov" = "true" ]; then
isbdf="false"
for i in $(${HOST_CMD} lspci -D | grep "storage controller" | grep "1ded\|Alibaba" | awk '{print $1}' | sed -n '/0$/p');
for i in $(host_cmd lspci -D | grep "storage controller" | grep "1ded\|Alibaba" | awk '{print $1}' | sed -n '/0$/p');
do
out=`${HOST_CMD} lspci -s $i -v`;
if [[ $out == *"Single Root I/O Virtualization"* ]]; then
out=$(host_cmd lspci -s "$i" -v);
if [[ "$out" = *"Single Root I/O Virtualization"* ]]; then
isbdf="true"
break
fi
Expand All @@ -273,15 +269,14 @@ if ([ "$DISK_BDF_ENABLE" = "true" ] && [ "$run_disk" = "true" ]) || [ "$run_pov"
echo "isbdf node: $isbdf"
if [ $isbdf = "true" ]; then
echo "start install vfhp"
${HOST_CMD} yum install -y "http://yum.tbsite.net/taobao/7/$ARCH/current/iohub-vfhp-helper/iohub-vfhp-helper-0.1.9-20231121192230.$ARCH.rpm"
if [ $? -ne 0 ]; then
${HOST_CMD} yum install -y "https://iohub-vfhp-helper.oss-rg-china-mainland.aliyuncs.com/iohub-vfhp-helper-0.1.9-20231121192230.$ARCH.rpm"
if ! host_cmd yum install -y "http://yum.tbsite.net/taobao/7/$ARCH/current/iohub-vfhp-helper/iohub-vfhp-helper-0.1.9-20231121192230.$ARCH.rpm"; then
host_cmd yum install -y "https://iohub-vfhp-helper.oss-rg-china-mainland.aliyuncs.com/iohub-vfhp-helper-0.1.9-20231121192230.$ARCH.rpm"
fi
# takes 10s
output=`${HOST_CMD} iohub-vfhp-helper -s`
if [[ $output == *"backend support auto vf hotplug."* ]]; then
output=$(host_cmd iohub-vfhp-helper -s)
if [[ "$output" == *"backend support auto vf hotplug."* ]]; then
echo "backend support auto vf hotplugin"
${HOST_CMD} sudo service iohub-vfhp-helper start
host_cmd sudo service iohub-vfhp-helper start
else
echo "backend not support auto vf hotplugin"
fi
Expand Down Expand Up @@ -309,35 +304,35 @@ if [ "$ARCH" = "x86_64" ] && [ "$run_nas" = "true" ]; then
if [ $install_utils = "true" ]; then
# cpfs-nas nas-rich-client common rpm
echo "installing aliyun-alinas-utils"
if [ "$HAS_YUM" = "true" ]; then
if [ "$SUPPORT_RPM" = "true" ]; then
PKG=aliyun-alinas-utils-1.1-8.20240527201444.2012cc.al7.noarch.rpm
elif [ "$HAS_APT" = "true" ]; then
elif [ "$PKG_MGR" = "apt" ]; then
PKG=aliyun-alinas-utils-1.1-8.deb
else
echo "WARN: no supported package manager found, skip install aliyun-alinas-utils"
install_utils="false"
fi
if [ $install_utils = "true" ]; then
install_package $PKG || exit $?
install_package "$PKG" || exit $?
fi
fi

if [ $host_os != "alinux2" ] && [ $install_efc = "true" ]; then
if { [ "$OS_RELEASE_ID" != "alinux" ] || [[ "$OS_RELEASE_VERSION_ID" != 2.* ]]; } && [ $install_efc = "true" ]; then
echo "WARN: skip install efc because host os is not alinux2"
install_efc="false"
fi
if [ $install_efc = "true" ]; then
# nas-rich-client rpm
echo "installing alinas-efc"
cp /root/alinas-efc-1.2-3.x86_64.rpm /host/etc/csi-tool/
${HOST_CMD} yum install -y /etc/csi-tool/alinas-efc-1.2-3.x86_64.rpm
host_cmd yum install -y /etc/csi-tool/alinas-efc-1.2-3.x86_64.rpm

echo "checking alinas-efc-1.2-3.x86_64 installed"
${HOST_CMD} rpm -q alinas-efc-1.2-3.x86_64 || exit 1
host_cmd rpm -q alinas-efc-1.2-3.x86_64 || exit 1
echo "starting aliyun-alinas-mount-watchdog"
${HOST_CMD} systemctl start aliyun-alinas-mount-watchdog || exit 1
host_cmd systemctl start aliyun-alinas-mount-watchdog || exit 1
fi
fi

# place it here to remove leftover from previous version
rm -rf /host/etc/csi-tool/*.{rpm,deb}
rm -rf /host/etc/csi-tool/*.rpm /host/etc/csi-tool/*.deb
Loading