From fbcedd67b946634ec1605e3dba3ecd5a2069dc6f Mon Sep 17 00:00:00 2001 From: lixiaoyuner <35456895+lixiaoyuner@users.noreply.github.com> Date: Tue, 25 Jul 2023 07:44:59 +0800 Subject: [PATCH] Add k8s master code new (#15716) Why I did it Currently, k8s master image is generated from a separate branch which we created by ourselves, not release ones. We need to commit these k8s master related code to master branch for a better way to do k8s master image build out. Work item tracking Microsoft ADO (number only): 19998138 How I did it Install k8s dashboard docker images Install geneva mds and mdsd and fluentd docker images and tag them as latest, tagging latest will help create container always with the latest version Install azure-storage-blob and azure-identity, this will help do etcd backup and restore. Install kubernetes python client packages, this will help read worker and container state, we can send these metric to Geneva. Remove mdm debian package, will replace it with the mdm docker image Add k8s master entrance script, this script will be called by rc-local service when system startup. we have some master systemd services in compute-move repo, when VMM service create master VM, VMM will copy all master service files inside VM, the entrance script will setup all services according to the service files. When the entrance script content changed, the PR build will set include_kubernetes_master=y to help do validation for k8s master related code change. The default value of include_kubernetes_master should be always n for public master branch. We will generate master image from internal master branch How to verify it Build with INCLUDE_KUBERNETES_MASTER = y --- .azure-pipelines/azure-pipelines-build.yml | 4 ++ .azure-pipelines/template-skipvstest.yml | 7 +++ build_debian.sh | 9 ---- .../build_templates/sonic_debian_extension.j2 | 20 +++++++-- .../kubernetes_master_entrance.service | 13 ++++++ .../kubernetes/kubernetes_master_entrance.sh | 45 +++++++++++++++++++ rules/config | 13 +++++- slave.mk | 5 +++ 8 files changed, 102 insertions(+), 14 deletions(-) create mode 100644 files/image_config/kubernetes/kubernetes_master_entrance.service diff --git a/.azure-pipelines/azure-pipelines-build.yml b/.azure-pipelines/azure-pipelines-build.yml index 67cfeaca95f4..df51700c4f20 100644 --- a/.azure-pipelines/azure-pipelines-build.yml +++ b/.azure-pipelines/azure-pipelines-build.yml @@ -130,6 +130,10 @@ jobs: make $BUILD_OPTIONS ENABLE_ASAN=y target/docker-sonic-vs.gz mv target/docker-sonic-vs.gz target/docker-sonic-vs-asan.gz fi + if [ $(K8S_OPTIONS) == 'INCLUDE_KUBERNETES_MASTER=y' ]; then + make $BUILD_OPTIONS $(K8S_OPTIONS) target/sonic-vs.img.gz + mv target/sonic-vs.img.gz target/sonic-vs-k8s.img.gz + fi make $BUILD_OPTIONS target/docker-sonic-vs.gz target/sonic-vs.img.gz target/docker-ptf.gz make $BUILD_OPTIONS target/docker-ptf-sai.gz if [ $(Build.Reason) != 'PullRequest' ];then diff --git a/.azure-pipelines/template-skipvstest.yml b/.azure-pipelines/template-skipvstest.yml index b47f17ce6e81..90d6fb6f7b49 100644 --- a/.azure-pipelines/template-skipvstest.yml +++ b/.azure-pipelines/template-skipvstest.yml @@ -3,6 +3,13 @@ steps: - script: | set -ex tar_branch=origin/$(System.PullRequest.TargetBranch) + # Check if k8s master entrance script is changed + k8s_master_changed=$(git diff $tar_branch..HEAD --name-only | grep -F files/image_config/kubernetes/kubernetes_master_entrance.sh) + if [ -z "$k8s_master_changed" ]; then + echo "##vso[task.setvariable variable=K8S_OPTIONS;]INCLUDE_KUBERNETES_MASTER=n" + else + echo "##vso[task.setvariable variable=K8S_OPTIONS;]INCLUDE_KUBERNETES_MASTER=y" + fi git diff $tar_branch..HEAD --name-only | grep -v -f .azure-pipelines/vstest-exclude && exit 0 git diff $tar_branch..HEAD --name-only | grep -f .azure-pipelines/vstest-include && exit 0 set +x diff --git a/build_debian.sh b/build_debian.sh index f71bdd9fd0fb..d04b8d0b3c17 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -295,17 +295,8 @@ then echo '[INFO] Install kubernetes master' install_kubernetes ${MASTER_KUBERNETES_VERSION} - sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -fsSL \ - https://packages.microsoft.com/keys/microsoft.asc | \ - sudo LANG=C chroot $FILESYSTEM_ROOT apt-key add - - sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -fsSL \ - https://packages.microsoft.com/keys/msopentech.asc | \ - sudo LANG=C chroot $FILESYSTEM_ROOT apt-key add - - echo "deb [arch=amd64] https://packages.microsoft.com/repos/azurecore-debian $IMAGE_DISTRO main" | \ - sudo tee $FILESYSTEM_ROOT/etc/apt/sources.list.d/azure.list sudo LANG=C chroot $FILESYSTEM_ROOT apt-get update sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install hyperv-daemons gnupg xmlstarlet - sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install metricsext2 sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y remove gnupg sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -o /tmp/cri-dockerd.deb -fsSL \ https://github.com/Mirantis/cri-dockerd/releases/download/v${MASTER_CRI_DOCKERD}/cri-dockerd_${MASTER_CRI_DOCKERD}.3-0.debian-${IMAGE_DISTRO}_amd64.deb diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2 index 29510542a2f2..d8399d25cb46 100644 --- a/files/build_templates/sonic_debian_extension.j2 +++ b/files/build_templates/sonic_debian_extension.j2 @@ -790,15 +790,27 @@ sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIV sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull k8s.gcr.io/kube-proxy:${MASTER_KUBERNETES_CONTAINER_IMAGE_VERSION} sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull k8s.gcr.io/coredns/coredns:${MASTER_COREDNS_VERSION} sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull k8s.gcr.io/etcd:${MASTER_ETCD_VERSION} +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull kubernetesui/metrics-scraper:${MASTER_UI_METRIC_VERSION} +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull kubernetesui/dashboard:${MASTER_UI_DASH_VERSION} +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull linuxgeneva-microsoft.azurecr.io/distroless/genevamdm:${MASTER_MDM_VERSION} +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS tag linuxgeneva-microsoft.azurecr.io/distroless/genevamdm:${MASTER_MDM_VERSION} linuxgeneva-microsoft.azurecr.io/distroless/genevamdm:latest +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull linuxgeneva-microsoft.azurecr.io/distroless/genevamdsd:${MASTER_MDS_VERSION} +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS tag linuxgeneva-microsoft.azurecr.io/distroless/genevamdsd:${MASTER_MDS_VERSION} linuxgeneva-microsoft.azurecr.io/distroless/genevamdsd:latest +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull linuxgeneva-microsoft.azurecr.io/distroless/genevafluentd_td-agent:${MASTER_FLUENTD_VERSION} +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS tag linuxgeneva-microsoft.azurecr.io/distroless/genevafluentd_td-agent:${MASTER_FLUENTD_VERSION} linuxgeneva-microsoft.azurecr.io/distroless/genevafluentd_td-agent:latest echo "kubernetes master docker images pull complete" -# Install python package for mdm service usage +# Install python packages for mdm metrics collection service usage sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install psutil sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install statsd -sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable mdm.service +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install kubernetes +# Install python packages to upload and download etcd backup files for backup and restore service usage +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install azure-storage-blob azure-identity sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable kubelet.service # Add kubernetes master entrance -sudo cp files/image_config/kubernetes/kubernetes_master_entrance.sh $FILESYSTEM_ROOT/usr/sbin/ -sudo sed -i '/^exit 0/i\bash /usr/sbin/kubernetes_master_entrance.sh' $FILESYSTEM_ROOT/etc/rc.local +sudo cp files/image_config/kubernetes/kubernetes_master_entrance.service ${FILESYSTEM_ROOT_USR_LIB_SYSTEMD_SYSTEM}/ +sudo cp files/image_config/kubernetes/kubernetes_master_entrance.sh $FILESYSTEM_ROOT/usr/bin/ +sudo chmod +x $FILESYSTEM_ROOT/usr/bin/kubernetes_master_entrance.sh +sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable kubernetes_master_entrance.service {% endif %} {% macro get_install_options(set_owner, enabled) -%} diff --git a/files/image_config/kubernetes/kubernetes_master_entrance.service b/files/image_config/kubernetes/kubernetes_master_entrance.service new file mode 100644 index 000000000000..3f407cf618ae --- /dev/null +++ b/files/image_config/kubernetes/kubernetes_master_entrance.service @@ -0,0 +1,13 @@ +[Unit] +Description=Setup k8s master services +Requires=rc-local.service +After=rc-local.service +Before=sonic.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/bin/kubernetes_master_entrance.sh + +[Install] +WantedBy=multi-user.target diff --git a/files/image_config/kubernetes/kubernetes_master_entrance.sh b/files/image_config/kubernetes/kubernetes_master_entrance.sh index b7695a61cae5..d73f4c4cb13c 100644 --- a/files/image_config/kubernetes/kubernetes_master_entrance.sh +++ b/files/image_config/kubernetes/kubernetes_master_entrance.sh @@ -1,2 +1,47 @@ +#!/bin/bash + # This script is for kubernetes master image usage # Will mount kubernetes master disk and execute kubernetes entrance script + +set -x + +# mount disk from host +mount_point="/from_host" +disk_by_path_dir="/dev/disk/by-path" +# We can't ensure the mounted disk name is sda or sdb +# Currently we specify the disk logic unit number to 2 when create master VM +# We find the correct disk by the disk logic unit number from by-path directory +target_str="lun-2-part1" +disk_path_file=$(ls $disk_by_path_dir | grep $target_str) + +# Check whether the disk path file exists +if [ -z "$disk_path_file" ]; then + echo "Error: Disk path file not found." + exit 1 +fi + +# Check the number of lines returned +line_count=$(echo "$disk_path_file" | wc -l) + +# If there are multiple lines, exit with status 1 +if [ "$line_count" -ne 1 ]; then + echo "Error: multiple disk path files found." + exit 1 +fi + +disk="${disk_by_path_dir}/${disk_path_file}" +mkdir -p $mount_point +mount $disk $mount_point + +# check whether it is the first time to boot +first_boot_flag_file="/from_host/first_boot_flag" +if [ -f $first_boot_flag_file ]; then + exit 0 +fi +touch $first_boot_flag_file + +# execute entrance script +init_file_name="entrance.sh" +init_file=${mount_point}/${init_file_name} +chmod +x $init_file +source $init_file diff --git a/rules/config b/rules/config index 04a5b53584d1..801ab9f79e0f 100644 --- a/rules/config +++ b/rules/config @@ -199,12 +199,23 @@ INCLUDE_KUBERNETES_MASTER ?= n # MASTER_PAUSE_VERSION - version of pause container image # MASTER_COREDNS_VERSION - version of coredns container image # MASTER_ETCD_VERSION = version of etcd container image +# MASTER_CRI_DOCKERD = version of cri-dockerd container image +# MASTER_UI_METRIC_VERSION = version of k8s metrics server container image +# MASTER_UI_DASH_VERSION = version of k8s dashboard container image +# MASTER_MDM_VERSION = version of mdm container image +# MASTER_MDS_VERSION = version of mds container image +# MASTER_FLUENTD_VERSION = version of fluentd container image MASTER_KUBERNETES_VERSION = 1.22.2-00 MASTER_KUBERNETES_CONTAINER_IMAGE_VERSION = v1.22.2 MASTER_PAUSE_VERSION = 3.5 MASTER_COREDNS_VERSION = v1.8.4 MASTER_ETCD_VERSION = 3.5.0-0 -MASTER_CRI_DOCKERD = 0.2.5 +MASTER_CRI_DOCKERD = 0.3.1 +MASTER_UI_METRIC_VERSION = v1.0.8 +MASTER_UI_DASH_VERSION = v2.7.0 +MASTER_MDM_VERSION = 2.2023.505.1124-45da18-20230505t1700 +MASTER_MDS_VERSION = mariner_20230517.1 +MASTER_FLUENTD_VERSION = mariner_20230517.1 # SONIC_ENABLE_IMAGE_SIGNATURE - enable image signature # To not use the auto-generated self-signed certificate, the required files to sign the image as below: diff --git a/slave.mk b/slave.mk index 045b3d076aff..300f6d3ff98b 100644 --- a/slave.mk +++ b/slave.mk @@ -1448,6 +1448,11 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ MASTER_COREDNS_VERSION=$(MASTER_COREDNS_VERSION) \ MASTER_ETCD_VERSION=$(MASTER_ETCD_VERSION) \ MASTER_CRI_DOCKERD=$(MASTER_CRI_DOCKERD) \ + MASTER_UI_METRIC_VERSION=$(MASTER_UI_METRIC_VERSION) \ + MASTER_UI_DASH_VERSION=$(MASTER_UI_DASH_VERSION) \ + MASTER_MDM_VERSION=$(MASTER_MDM_VERSION) \ + MASTER_MDS_VERSION=$(MASTER_MDS_VERSION) \ + MASTER_FLUENTD_VERSION=$(MASTER_FLUENTD_VERSION) \ ./build_debian.sh $(LOG) USERNAME="$(USERNAME)" \