Add k8s master code new (sonic-net#15716)

Why I did it
Currently, k8s master image is generated from a separate branch which we created by ourselves, not release ones. We need to commit these k8s master related code to master branch for a better way to do k8s master image build out.

Work item tracking
Microsoft ADO (number only):
19998138
How I did it
Install k8s dashboard docker images
Install geneva mds and mdsd and fluentd docker images and tag them as latest, tagging latest will help create container always with the latest version
Install azure-storage-blob and azure-identity, this will help do etcd backup and restore.
Install kubernetes python client packages, this will help read worker and container state, we can send these metric to Geneva.
Remove mdm debian package, will replace it with the mdm docker image
Add k8s master entrance script, this script will be called by rc-local service when system startup. we have some master systemd services in compute-move repo, when VMM service create master VM, VMM will copy all master service files inside VM, the entrance script will setup all services according to the service files.
When the entrance script content changed, the PR build will set include_kubernetes_master=y to help do validation for k8s master related code change. The default value of include_kubernetes_master should be always n for public master branch. We will generate master image from internal master branch
How to verify it
Build with INCLUDE_KUBERNETES_MASTER = y

.azure-pipelines/azure-pipelines-build.yml

@@ -130,6 +130,10 @@ jobs:
               make $BUILD_OPTIONS ENABLE_ASAN=y target/docker-sonic-vs.gz
               mv target/docker-sonic-vs.gz target/docker-sonic-vs-asan.gz
             fi
+            if [ $(K8S_OPTIONS) == 'INCLUDE_KUBERNETES_MASTER=y' ]; then
+              make $BUILD_OPTIONS $(K8S_OPTIONS) target/sonic-vs.img.gz
+              mv target/sonic-vs.img.gz target/sonic-vs-k8s.img.gz
+            fi
             make $BUILD_OPTIONS target/docker-sonic-vs.gz target/sonic-vs.img.gz target/docker-ptf.gz
             make $BUILD_OPTIONS target/docker-ptf-sai.gz
             if [ $(Build.Reason) != 'PullRequest' ];then

.azure-pipelines/template-skipvstest.yml

@@ -3,6 +3,13 @@ steps:
   - script: |
       set -ex
       tar_branch=origin/$(System.PullRequest.TargetBranch)
+      # Check if k8s master entrance script is changed
+      k8s_master_changed=$(git diff $tar_branch..HEAD --name-only | grep -F files/image_config/kubernetes/kubernetes_master_entrance.sh)
+      if [ -z "$k8s_master_changed" ]; then
+        echo "##vso[task.setvariable variable=K8S_OPTIONS;]INCLUDE_KUBERNETES_MASTER=n"
+      else
+        echo "##vso[task.setvariable variable=K8S_OPTIONS;]INCLUDE_KUBERNETES_MASTER=y"
+      fi
       git diff $tar_branch..HEAD --name-only | grep -v -f .azure-pipelines/vstest-exclude && exit 0
       git diff $tar_branch..HEAD --name-only | grep -f .azure-pipelines/vstest-include && exit 0
       set +x

build_debian.sh

@@ -295,17 +295,8 @@ then
     echo '[INFO] Install kubernetes master'
     install_kubernetes ${MASTER_KUBERNETES_VERSION}
 
-    sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -fsSL \
-        https://packages.microsoft.com/keys/microsoft.asc | \
-        sudo LANG=C chroot $FILESYSTEM_ROOT apt-key add -
-    sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -fsSL \
-        https://packages.microsoft.com/keys/msopentech.asc | \
-        sudo LANG=C chroot $FILESYSTEM_ROOT apt-key add -
-    echo "deb [arch=amd64] https://packages.microsoft.com/repos/azurecore-debian $IMAGE_DISTRO main" | \
-        sudo tee $FILESYSTEM_ROOT/etc/apt/sources.list.d/azure.list
     sudo LANG=C chroot $FILESYSTEM_ROOT apt-get update
     sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install hyperv-daemons gnupg xmlstarlet
-    sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install metricsext2
     sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y remove gnupg
     sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT curl -o /tmp/cri-dockerd.deb -fsSL \
         https://github.com/Mirantis/cri-dockerd/releases/download/v${MASTER_CRI_DOCKERD}/cri-dockerd_${MASTER_CRI_DOCKERD}.3-0.debian-${IMAGE_DISTRO}_amd64.deb

files/build_templates/sonic_debian_extension.j2

@@ -790,15 +790,27 @@ sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIV
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull k8s.gcr.io/kube-proxy:${MASTER_KUBERNETES_CONTAINER_IMAGE_VERSION}
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull k8s.gcr.io/coredns/coredns:${MASTER_COREDNS_VERSION}
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull k8s.gcr.io/etcd:${MASTER_ETCD_VERSION}
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull kubernetesui/metrics-scraper:${MASTER_UI_METRIC_VERSION}
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull kubernetesui/dashboard:${MASTER_UI_DASH_VERSION}
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull linuxgeneva-microsoft.azurecr.io/distroless/genevamdm:${MASTER_MDM_VERSION}
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS tag linuxgeneva-microsoft.azurecr.io/distroless/genevamdm:${MASTER_MDM_VERSION} linuxgeneva-microsoft.azurecr.io/distroless/genevamdm:latest
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull linuxgeneva-microsoft.azurecr.io/distroless/genevamdsd:${MASTER_MDS_VERSION}
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS tag linuxgeneva-microsoft.azurecr.io/distroless/genevamdsd:${MASTER_MDS_VERSION} linuxgeneva-microsoft.azurecr.io/distroless/genevamdsd:latest
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS pull linuxgeneva-microsoft.azurecr.io/distroless/genevafluentd_td-agent:${MASTER_FLUENTD_VERSION}
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS tag linuxgeneva-microsoft.azurecr.io/distroless/genevafluentd_td-agent:${MASTER_FLUENTD_VERSION} linuxgeneva-microsoft.azurecr.io/distroless/genevafluentd_td-agent:latest
 echo "kubernetes master docker images pull complete"
-# Install python package for mdm service usage
+# Install python packages for mdm metrics collection service usage
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install psutil
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install statsd
-sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable mdm.service
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install kubernetes
+# Install python packages to upload and download etcd backup files for backup and restore service usage
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install azure-storage-blob azure-identity
 sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable kubelet.service
 # Add kubernetes master entrance
-sudo cp files/image_config/kubernetes/kubernetes_master_entrance.sh $FILESYSTEM_ROOT/usr/sbin/
-sudo sed -i '/^exit 0/i\bash /usr/sbin/kubernetes_master_entrance.sh' $FILESYSTEM_ROOT/etc/rc.local
+sudo cp files/image_config/kubernetes/kubernetes_master_entrance.service ${FILESYSTEM_ROOT_USR_LIB_SYSTEMD_SYSTEM}/
+sudo cp files/image_config/kubernetes/kubernetes_master_entrance.sh $FILESYSTEM_ROOT/usr/bin/
+sudo chmod +x $FILESYSTEM_ROOT/usr/bin/kubernetes_master_entrance.sh
+sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable kubernetes_master_entrance.service
 {% endif %}
 
 {% macro get_install_options(set_owner, enabled) -%}

files/image_config/kubernetes/kubernetes_master_entrance.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Setup k8s master services
+Requires=rc-local.service
+After=rc-local.service
+Before=sonic.target
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/bin/kubernetes_master_entrance.sh
+
+[Install]
+WantedBy=multi-user.target

files/image_config/kubernetes/kubernetes_master_entrance.sh

@@ -1,2 +1,47 @@
+#!/bin/bash
+
 # This script is for kubernetes master image usage
 # Will mount kubernetes master disk and execute kubernetes entrance script
+
+set -x
+
+# mount disk from host
+mount_point="/from_host"
+disk_by_path_dir="/dev/disk/by-path"
+# We can't ensure the mounted disk name is sda or sdb
+# Currently we specify the disk logic unit number to 2 when create master VM
+# We find the correct disk by the disk logic unit number from by-path directory
+target_str="lun-2-part1"
+disk_path_file=$(ls $disk_by_path_dir | grep $target_str)
+
+# Check whether the disk path file exists
+if [ -z "$disk_path_file" ]; then
+    echo "Error: Disk path file not found."
+    exit 1
+fi
+
+# Check the number of lines returned
+line_count=$(echo "$disk_path_file" | wc -l)
+
+# If there are multiple lines, exit with status 1
+if [ "$line_count" -ne 1 ]; then
+    echo "Error: multiple disk path files found."
+    exit 1
+fi
+
+disk="${disk_by_path_dir}/${disk_path_file}"
+mkdir -p $mount_point
+mount $disk $mount_point
+
+# check whether it is the first time to boot
+first_boot_flag_file="/from_host/first_boot_flag"
+if [ -f $first_boot_flag_file ]; then
+    exit 0
+fi
+touch $first_boot_flag_file
+
+# execute entrance script
+init_file_name="entrance.sh"
+init_file=${mount_point}/${init_file_name}
+chmod +x $init_file
+source $init_file

rules/config

@@ -199,12 +199,23 @@ INCLUDE_KUBERNETES_MASTER ?= n
 # MASTER_PAUSE_VERSION - version of pause container image
 # MASTER_COREDNS_VERSION - version of coredns container image
 # MASTER_ETCD_VERSION = version of etcd container image
+# MASTER_CRI_DOCKERD = version of cri-dockerd container image
+# MASTER_UI_METRIC_VERSION = version of k8s metrics server container image
+# MASTER_UI_DASH_VERSION = version of k8s dashboard container image
+# MASTER_MDM_VERSION = version of mdm container image
+# MASTER_MDS_VERSION = version of mds container image
+# MASTER_FLUENTD_VERSION = version of fluentd container image
 MASTER_KUBERNETES_VERSION = 1.22.2-00
 MASTER_KUBERNETES_CONTAINER_IMAGE_VERSION = v1.22.2
 MASTER_PAUSE_VERSION = 3.5
 MASTER_COREDNS_VERSION = v1.8.4
 MASTER_ETCD_VERSION = 3.5.0-0
-MASTER_CRI_DOCKERD = 0.2.5
+MASTER_CRI_DOCKERD = 0.3.1
+MASTER_UI_METRIC_VERSION = v1.0.8
+MASTER_UI_DASH_VERSION = v2.7.0
+MASTER_MDM_VERSION = 2.2023.505.1124-45da18-20230505t1700
+MASTER_MDS_VERSION = mariner_20230517.1
+MASTER_FLUENTD_VERSION = mariner_20230517.1
 
 # SONIC_ENABLE_IMAGE_SIGNATURE - enable image signature
 # To not use the auto-generated self-signed certificate, the required files to sign the image as below:

slave.mk

@@ -1448,6 +1448,11 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
 		MASTER_COREDNS_VERSION=$(MASTER_COREDNS_VERSION) \
 		MASTER_ETCD_VERSION=$(MASTER_ETCD_VERSION) \
 		MASTER_CRI_DOCKERD=$(MASTER_CRI_DOCKERD) \
+		MASTER_UI_METRIC_VERSION=$(MASTER_UI_METRIC_VERSION) \
+		MASTER_UI_DASH_VERSION=$(MASTER_UI_DASH_VERSION) \
+		MASTER_MDM_VERSION=$(MASTER_MDM_VERSION) \
+		MASTER_MDS_VERSION=$(MASTER_MDS_VERSION) \
+		MASTER_FLUENTD_VERSION=$(MASTER_FLUENTD_VERSION) \
 			./build_debian.sh $(LOG)
 
 		USERNAME="$(USERNAME)" \