feat: K8S集群在离线安装增加arm64架构支持

TencentBlueKing · Nov 4, 2024 · dc9f424 · dc9f424
1 parent 1fd53b6
commit dc9f424
Show file tree

Hide file tree

Showing 16 changed files with 279 additions and 141 deletions.
diff --git a/bcs-ops/Makefile b/bcs-ops/Makefile
@@ -3,28 +3,26 @@ VER = v0.0.2
 CURRENT_VERSION = release-$(VER)
 
 clean:
-	-rm ./bcs-ops-script-release-$(VER)*.tar.gz
+	-rm ./bcs-ops-script-release-$(VER)*.tgz
 	-rm ./bcs-ops-offline-release-$(VER)-k8s-*.tgz
 	-rm -r ./bcs-ops-offline
 	-rm MD5SUMS
 
 build:clean
 	find . -not -path "*/.git/*" -a -not -path "*/bcs-ops-offline/*" -a -not -path "*/charts/*" -a -not -path "*/functions/*" -a -not -path "*.yaml" -a -not -path "*.md" -type f -print0 | xargs -0 chmod 555
 	find ./functions/ -not -path "*/.git/*" -a -not -path "*/bcs-ops-offline/*" -a -not -path "*/charts/*"  -type f -print0 | xargs -0 chmod 444
-	tar -czvf bcs-ops-script-release-$(VER).tar.gz --exclude=bcs-ops-offline --exclude=\..* --exclude=.*tar.gz ./*
-	md5sum bcs-ops-script-release-$(VER).tar.gz >> MD5SUMS
+	tar -czvf bcs-ops-script-release-$(VER).tgz --exclude=bcs-ops-offline --exclude=\..* --exclude=.*tar.gz --exclude=.*tgz  --exclude=version-.* --exclude=.*xz ./*
+	md5sum bcs-ops-script-release-$(VER).tgz >> MD5SUMS
 
-build_offline_pkg:clean
-	find . -not -path "*/.git/*" -a -not -path "*/bcs-ops-offline/*" -a -not -path "*/charts/*" -a -not -path "*/functions/*" -a -not -path "*.yaml" -a -not -path "*.md" -type f -print0 | xargs -0 chmod 555
-	find ./functions/ -not -path "*/.git/*" -a -not -path "*/bcs-ops-offline/*" -a -not -path "*/charts/*" -type f -print0 | xargs -0 chmod 444
-	tar -czvf bcs-ops-script-release-$(VER).tar.gz --exclude=bcs-ops-offline --exclude=\..* --exclude=.*tar.gz ./*
-	md5sum bcs-ops-script-release-$(VER).tar.gz >> MD5SUMS
+build_offline_1:build
 	./offline_package.sh env/offline-manifest.yaml
-	for f in $(find bcs-ops-offline -iname bcs-ops-offline-*.tgz -type f);do
-		mv $f bcs-ops-offline-release-$(VER)-k8s-${f##*-}
+	mv ./bcs-ops-offline/*.tgz ./
+
+build_offline_pkg:build_offline_1
+	for f in $(shell ls bcs-ops-offline*.tgz);do \
+		mv $$f bcs-ops-offline-release-$(VER)-k8s-$${f##*-offline-}; \
 	done
 	md5sum bcs-ops-offline-release-$(VER)-k8s-*.tgz >> MD5SUMS
 
-
 build_image:clean
-	docker build -f docker/Dockerfile -t bcs-ops-upload:test .
+	docker build -f docker/Dockerfile -t bcs-ops-upload:test .
diff --git a/bcs-ops/bcs-ops b/bcs-ops/bcs-ops
@@ -35,7 +35,7 @@ Usage:
     [ -h --help -?     show usage ]
     [ -v -V --version  show script version]
     [ -i --install     support: ${INSTALL_PROJECTS[*]}]
-    [ -r --render      suppport: ${RENDER_PROJECTS[*]}]
+    [ -r --render      support: ${RENDER_PROJECTS[*]}]
     [ -c --clean       support: ${CLEAN_PROJECTS[*]}]
     [ -e --check       support: ${CHECK_PROJECTS[*]}]
 EOF

diff --git a/bcs-ops/install_master.sh b/bcs-ops/install_master.sh
@@ -60,6 +60,9 @@ safe_source "${ROOT_DIR}/env/bcs.env"
 # wait to check kubelet start
 sleep 30
 if [[ -z ${MASTER_JOIN_CMD:-} ]]; then
+  if [[ -n ${JOIN_CMD:-} ]]; then
+    utils::log "FATAL" "install master need to set MASTER_JOIN_CMD but JOIN_CMD is set"
+  fi
   if systemctl is-active kubelet.service -q; then
     utils::log "WARN" "kubelet service is active now, skip kubeadm init"
   else
@@ -84,8 +87,8 @@ if [[ -z ${MASTER_JOIN_CMD:-} ]]; then
       "${ROOT_DIR}"/k8s/operate_kube_vip apply
     elif [[ ${APISERVER_HA_MODE} == "bcs-apiserver-proxy" ]]; then
       "${ROOT_DIR}"/k8s/operate_bap apply
-    else
-      "${ROOT_DIR}"/k8s/operate_bap apply
+    elif [[ ${APISERVER_HA_MODE} == "third-party" ]];then
+      utils::log "INFO" "use third-party lb, do nothing"
     fi
   fi
 
@@ -94,6 +97,9 @@ if [[ -z ${MASTER_JOIN_CMD:-} ]]; then
       utils::log "FATAL" "fail to apply multus"
     fi
   fi
+
+  # create etcd secret
+  kubectl create secret generic etcd-client-cert --from-file=etcd-ca=/etc/kubernetes/pki/ca.crt --from-file=etcd-client-key=/etc/kubernetes/pki/apiserver-etcd-client.key --from-file=etcd-client=/etc/kubernetes/pki/apiserver-etcd-client.crt -n kube-system
 else
   if systemctl is-active kubelet.service -q; then
     utils::log "WARN" "kubelet service is active now, skip kubeadm join"

diff --git a/bcs-ops/k8s/install_containerd b/bcs-ops/k8s/install_containerd
@@ -85,7 +85,7 @@ _curl_containerd() {
   name="containerd"
   ver=$(awk '/version: \"'"${K8S_VER}"'\"/{f=1;next} f && /'"${name}"':/{gsub("\"","",$2);print $2;exit}' "${ROOT_DIR}"/env/offline-manifest.yaml)
   file="${name}-${ver}.tgz"
-  url=${REPO_URL}/${file}
+  url=${REPO_URL}/containerd/${file}
   if curl -sSfL "${url}" -o "${bin_path}/${file}" -m "360"; then
     utils::log "INFO" "Downloaded ${url}"
   else
@@ -95,7 +95,7 @@ _curl_containerd() {
   name="runc"
   ver=$(awk '/version: \"'"${K8S_VER}"'\"/{f=1;next} f && /'"${name}"':/{gsub("\"","",$2);print $2;exit}' "${ROOT_DIR}"/env/offline-manifest.yaml)
   file="${name}-${ver}.tgz"
-  url="${REPO_URL}/${file}"
+  url="${REPO_URL}/runc/${file}"
   if curl -sSfL "${url}" -o "${bin_path}/${file}" -m "360"; then
     utils::log "INFO" "Downloaded ${url}"
   else
@@ -109,15 +109,50 @@ _offline_containerd() {
   local bin_path tar_name
   bin_path=${ROOT_DIR}/version-${K8S_VER}/bin-tools/
 
-  tar_name=$(find "$bin_path" -iname "containerd-*.tgz" -type f | head -1)
+  tar_name=$(find "$bin_path" -iname "containerd-*-${ARCH}.tgz" -type f | head -1)
   if [[ -z ${tar_name} ]]; then
     utils::log "FATAL" "can't find docker installation package in ${bin_path}"
   else
     tar xvzf "${tar_name}" -C /usr/local/bin/ --strip-components=1 bin/
-    tar xvzf "${tar_name}" -C /etc/systemd/system/ --strip-components=1 systemd/
+    cat <<EOF >/etc/systemd/system/containerd.service
+[Unit]
+Description=Docker Application Container Engine
+Documentation=https://docs.docker.com
+After=network-online.target docker.socket firewalld.service
+Wants=network-online.target
+Requires=docker.socket
+
+[Service]
+Type=notify
+# the default is not to use systemd for cgroups because the delegate issues still
+# exists and systemd currently does not support the cgroup feature set required
+# for containers run by docker
+ExecStart=/usr/bin/dockerd -H fd://
+ExecReload=/bin/kill -s HUP \$MAINPID
+LimitNOFILE=1048576
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNPROC=infinity
+LimitCORE=infinity
+# Uncomment TasksMax if your systemd version supports it.
+# Only systemd 226 and above support this version.
+#TasksMax=infinity
+TimeoutStartSec=0
+# set delegate yes so that systemd does not reset the cgroups of docker containers
+Delegate=yes
+# kill only the docker process, not all processes in the cgroup
+KillMode=process
+# restart the docker process if it exits prematurely
+Restart=on-failure
+StartLimitBurst=3
+StartLimitInterval=60s
+
+[Install]
+WantedBy=multi-user.target
+EOF
   fi
 
-  tar_name=$(find "$bin_path" -iname "runc-*.tgz" -type f | head -1)
+  tar_name=$(find "$bin_path" -iname "runc-*-${ARCH}.tgz" -type f | head -1)
   if [[ -z ${tar_name} ]]; then
     utils::log "FATAL" "can't find docker installation package in ${bin_path}"
   else

diff --git a/bcs-ops/k8s/install_docker b/bcs-ops/k8s/install_docker
@@ -91,7 +91,17 @@ _curl_docker() {
   name="docker"
   ver=$(awk '/version: \"'"${K8S_VER}"'\"/{f=1;next} f && /'"${name}"':/{gsub("\"","",$2);print $2;exit}' "${ROOT_DIR}"/env/offline-manifest.yaml)
   file="${name}-${ver}.tgz"
-  url=${REPO_URL}/${file}
+  url=${REPO_URL}/docker/${file}
+  if curl -sSfL "${url}" -o "${bin_path}/${file}" -m "360"; then
+    utils::log "INFO" "Downloaded ${url}"
+  else
+    utils::log "ERROR" "fail to download ${url}"
+  fi
+
+  name="runc"
+  ver=$(awk '/version: \"'"${K8S_VER}"'\"/{f=1;next} f && /'"${name}"':/{gsub("\"","",$2);print $2;exit}' "${ROOT_DIR}"/env/offline-manifest.yaml)
+  file="${name}-${ver}.tgz"
+  url="${REPO_URL}/runc/${file}"
   if curl -sSfL "${url}" -o "${bin_path}/${file}" -m "360"; then
     utils::log "INFO" "Downloaded ${url}"
   else
@@ -104,13 +114,63 @@ _curl_docker() {
 _offline_docker() {
   local bin_path tar_name
   bin_path=${ROOT_DIR}/version-${K8S_VER}/bin-tools/
-  tar_name=$(find "$bin_path" -iname "docker-*.tgz" -type f | head -1)
+  tar_name=$(find "$bin_path" -iname "docker-*-${ARCH}.tgz" -type f | head -1)
 
   if [[ -z ${tar_name} ]]; then
     utils::log "FATAL" "can't find docker installation package in ${bin_path}"
   else
     tar xvzf "${tar_name}" -C /usr/bin/ --strip-components=1 bin/
-    tar xvzf "${tar_name}" -C /etc/systemd/system/ --strip-components=1 systemd/
+#    tar xvzf "${tar_name}" -C /etc/systemd/system/ --strip-components=1 systemd/
+    cat <<EOF >/etc/systemd/system/docker.socket
+[Unit]
+Description=Docker Socket for the API
+PartOf=docker.service
+
+[Socket]
+ListenStream=/var/run/docker.sock
+SocketMode=0660
+SocketUser=root
+SocketGroup=docker
+
+[Install]
+WantedBy=sockets.target
+EOF
+    cat <<EOF >/etc/systemd/system/docker.service
+[Unit]
+Description=Docker Application Container Engine
+Documentation=https://docs.docker.com
+After=network-online.target docker.socket firewalld.service
+Wants=network-online.target
+Requires=docker.socket
+
+[Service]
+Type=notify
+# the default is not to use systemd for cgroups because the delegate issues still
+# exists and systemd currently does not support the cgroup feature set required
+# for containers run by docker
+ExecStart=/usr/bin/dockerd -H fd://
+ExecReload=/bin/kill -s HUP \$MAINPID
+LimitNOFILE=1048576
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNPROC=infinity
+LimitCORE=infinity
+# Uncomment TasksMax if your systemd version supports it.
+# Only systemd 226 and above support this version.
+#TasksMax=infinity
+TimeoutStartSec=0
+# set delegate yes so that systemd does not reset the cgroups of docker containers
+Delegate=yes
+# kill only the docker process, not all processes in the cgroup
+KillMode=process
+# restart the docker process if it exits prematurely
+Restart=on-failure
+StartLimitBurst=3
+StartLimitInterval=60s
+
+[Install]
+WantedBy=multi-user.target
+EOF
   fi
 
   # docker.sock need docker group
@@ -120,6 +180,13 @@ _offline_docker() {
     utils::log "INFO" "creating docker group"
     groupadd docker
   fi
+
+  tar_name=$(find "$bin_path" -iname "runc-*-${ARCH}.tgz" -type f | head -1)
+  if [[ -z ${tar_name} ]]; then
+    utils::log "FATAL" "can't find docker installation package in ${bin_path}"
+  else
+    tar xvzf "${tar_name}" -C /usr/bin/ --strip-components=1 bin/
+  fi
 }
 
 # ToDo: config separte

diff --git a/bcs-ops/k8s/install_k8s_tools b/bcs-ops/k8s/install_k8s_tools
@@ -92,7 +92,7 @@ _curl_k8s() {
     '/version: \"'"${K8S_VER}"'\"/{f=1;next} f && /'"${name}"':/{gsub("\"","",$2);print $2;exit}' \
     "${ROOT_DIR}"/env/offline-manifest.yaml)
   file="${name}-${ver}.tgz"
-  url=${REPO_URL:-}/${file}
+  url=${REPO_URL:-}/k8s/${file}
   if curl -sSfL "${url}" -o "${bin_path}/${file}" -m "360"; then
     utils::log "INFO" "Downloaded ${url}"
   else
@@ -104,7 +104,7 @@ _curl_k8s() {
     '/version: \"'"${K8S_VER}"'\"/{f=1;next} f && /'"${name}"':/{gsub("\"","",$2);print $2;exit}' \
     "${ROOT_DIR}"/env/offline-manifest.yaml)
   file="${name}-${ver}.tgz"
-  url="${REPO_URL}/${file}"
+  url="${REPO_URL}/crictl/${file}"
   if curl -sSfL "${url}" -o "${bin_path}/${file}" -m "360"; then
     utils::log "INFO" "Downloaded ${url}"
   else
@@ -116,7 +116,7 @@ _curl_k8s() {
     '/version: \"'"${K8S_VER}"'\"/{f=1;next} f && /'"${name}"':/{gsub("\"","",$2);print $2;exit}' \
     "${ROOT_DIR}"/env/offline-manifest.yaml)
   file="${name}-${ver}.tgz"
-  url="${REPO_URL}/${file}"
+  url="${REPO_URL}/cni-plugins/${file}"
   if curl -sSfL "${url}" -o "${bin_path}/${file}" -m "360"; then
     utils::log "INFO" "Downloaded ${url}"
   else
@@ -130,7 +130,7 @@ _offline_k8s() {
   local bin_path tar_name
   bin_path=${ROOT_DIR}/version-${K8S_VER}/bin-tools/
 
-  tar_name=$(find "$bin_path" -iname "k8s-*.tgz" -type f | head -1)
+  tar_name=$(find "$bin_path" -iname "k8s-*-${ARCH}.tgz" -type f | head -1)
   if [[ -z ${tar_name} ]]; then
     utils::log "FATAL" "can't find docker installation package in ${bin_path}"
   else
@@ -141,14 +141,14 @@ _offline_k8s() {
       --strip-components=1 systemd/10-kubeadm.conf
   fi
 
-  tar_name=$(find "$bin_path" -iname "crictl-*.tgz" -type f | head -1)
+  tar_name=$(find "$bin_path" -iname "crictl-*-${ARCH}.tgz" -type f | head -1)
   if [[ -z ${tar_name} ]]; then
     utils::log "FATAL" "can't find docker installation package in ${bin_path}"
   else
     tar xvzf "${tar_name}" -C /usr/bin/ --strip-components=1 bin/
   fi
 
-  tar_name=$(find "$bin_path" -iname "cni-plugins-*.tgz" -type f | head -1)
+  tar_name=$(find "$bin_path" -iname "cni-plugins-*-${ARCH}.tgz" -type f | head -1)
   if [[ -z ${tar_name} ]]; then
     utils::log "FATAL" "can't find docker installation package in ${bin_path}"
   else

diff --git a/bcs-ops/k8s/optimize_k8s b/bcs-ops/k8s/optimize_k8s
@@ -129,6 +129,11 @@ else
   fi
 fi
 
+#add label
+yq -i '.metadata.labels.k8s-app="kube-controller-manager"' /etc/kubernetes/manifests/kube-controller-manager.yaml
+yq -i '.metadata.labels.k8s-app="kube-apiserver"' /etc/kubernetes/manifests/kube-apiserver.yaml
+yq -i '.metadata.labels.k8s-app="kube-scheduler"' /etc/kubernetes/manifests/kube-scheduler.yaml
+
 for pod_file in "${pod_files[@]}"; do
   cp "${ROOT_DIR}/${pod_file}" /etc/kubernetes/manifests/
 done