ubuntu-provision.template.yaml

#cloud-config

ssh_pwauth: false

ssh_authorized_keys:
write_files:
- path: '/opt/bin/health-monitor.sh'
  permissions: '0755'
  encoding: 'b64'
  content: |-
    #!/usr/bin/env bash
    # Copyright 2016 The Kubernetes Authors.
    #
    # Licensed under the Apache License, Version 2.0 (the "License");
    # you may not use this file except in compliance with the License.
    # You may obtain a copy of the License at
    #
    #     http://www.apache.org/licenses/LICENSE-2.0
    #
    # Unless required by applicable law or agreed to in writing, software
    # distributed under the License is distributed on an "AS IS" BASIS,
    # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    # See the License for the specific language governing permissions and
    # limitations under the License.
    
    # This script is for master and node instance health monitoring, which is
    # packed in kube-manifest tarball. It is executed through a systemd service
    # in cluster/gce/gci/<master/node>.yaml. The env variables come from an env
    # file provided by the systemd service.
    
    # This script is a slightly adjusted version of
    # https://github.com/kubernetes/kubernetes/blob/e1a1aa211224fcd9b213420b80b2ae680669683d/cluster/gce/gci/health-monitor.sh
    # Adjustments are:
    # * Kubelet health port is 10248 not 10255
    # * Removal of all all references to the KUBE_ENV file
    
    set -o nounset
    set -o pipefail
    
    # We simply kill the process when there is a failure. Another systemd service will
    # automatically restart the process.
    function container_runtime_monitoring() {
    local -r max_attempts=5
    local attempt=1
    local -r container_runtime_name="${CONTAINER_RUNTIME_NAME:-containerd}"
    # We still need to use 'docker ps' when container runtime is "docker". This is because
    # dockershim is still part of kubelet today. When kubelet is down, crictl pods
    # will also fail, and docker will be killed. This is undesirable especially when
    # docker live restore is disabled.
    local healthcheck_command="docker ps"
    if [[ "${CONTAINER_RUNTIME:-containerd}" != "docker" ]]; then
    healthcheck_command="crictl pods"
    fi
    # Container runtime startup takes time. Make initial attempts before starting
    # killing the container runtime.
    until timeout 60 ${healthcheck_command} +IC9kZXYvbnVsbDsgZG8KICAgIGlmICgoYXR0ZW1wdCA9PSBtYXhfYXR0ZW1wdHMpKTsgdGhlbgogICAgICBlY2hvICJNYXggYXR0ZW1wdCAke21heF9hdHRlbXB0c30gcmVhY2hlZCEgUHJvY2VlZGluZyB0byBtb25pdG9yIGNvbnRhaW5lciBydW50aW1lIGhlYWx0aGluZXNzLiIKICAgICAgYnJlYWsKICAgIGZpCiAgICBlY2hvICIkYXR0ZW1wdCBpbml0aWFsIGF0dGVtcHQgXCIke2hlYWx0aGNoZWNrX2NvbW1hbmR9XCIhIFRyeWluZyBhZ2FpbiBpbiAkYXR0ZW1wdCBzZWNvbmRzLi4uIgogICAgc2xlZXAgIiQoKDIgKiogYXR0ZW1wdCsrKSkiCiAgZG9uZQogIHdoaWxlIHRydWU7IGRvCiAgICBpZiAhIHRpbWVvdXQgNjAgJHtoZWFsdGhjaGVja19jb21tYW5kfSA+IC9kZXYvbnVsbDsgdGhlbgogICAgICBlY2hvICJDb250YWluZXIgcnVudGltZSAke2NvbnRhaW5lcl9ydW50aW1lX25hbWV9IGZhaWxlZCEiCiAgICAgIGlmIFtbICIkY29udGFpbmVyX3J1bnRpbWVfbmFtZSIgPT0gImRvY2tlciIgXV07IHRoZW4KICAgICAgICAjIER1bXAgc3RhY2sgb2YgZG9ja2VyIGRhZW1vbiBmb3IgaW52ZXN0aWdhdGlvbi4KICAgICAgICAjIExvZyBmaWxlIG5hbWUgbG9va3MgbGlrZSBnb3JvdXRpbmUtc3RhY2tzLVRJTUVTVEFNUCBhbmQgd2lsbCBiZSBzYXZlZCB0bwogICAgICAgICMgdGhlIGV4ZWMgcm9vdCBkaXJlY3RvcnksIHdoaWNoIGlzIC92YXIvcnVuL2RvY2tlci8gb24gVWJ1bnR1IGFuZCBDT1MuCiAgICAgICAgcGtpbGwgLVNJR1VTUjEgZG9ja2VyZAogICAgICBmaQogICAgICBzeXN0ZW1jdGwga2lsbCAtLWtpbGwtd2hvPW1haW4gIiR7Y29udGFpbmVyX3J1bnRpbWVfbmFtZX0iCiAgICAgICMgV2FpdCBmb3IgYSB3aGlsZSwgYXMgd2UgZG9uJ3Qgd2FudCB0byBraWxsIGl0IGFnYWluIGJlZm9yZSBpdCBpcyByZWFsbHkgdXAuCiAgICAgIHNsZWVwIDEyMAogICAgZWxzZQogICAgICBzbGVlcCAiJHtTTEVFUF9TRUNPTkRTfSIKICAgIGZpCiAgZG9uZQp9CgpmdW5jdGlvbiBrdWJlbGV0X21vbml0b3JpbmcoKSB7CiAgZWNobyAiV2FpdCBmb3IgMiBtaW51dGVzIGZvciBrdWJlbGV0IHRvIGJlIGZ1bmN0aW9uYWwiCiAgc2xlZXAgMTIwCiAgbG9jYWwgLXIgbWF4X3NlY29uZHM9MTAKICBsb2NhbCBvdXRwdXQ9IiIKICB3aGlsZSB0cnVlOyBkbwogICAgbG9jYWwgZmFpbGVkPWZhbHNlCgogICAgaWYgam91cm5hbGN0bCAtdSBrdWJlbGV0IC1uIDEgfCBncmVwIC1xICJ1c2Ugb2YgY2xvc2VkIG5ldHdvcmsgY29ubmVjdGlvbiI7IHRoZW4KICAgICAgZmFpbGVkPXRydWUKICAgICAgZWNobyAiS3ViZWxldCBzdG9wcGVkIHBvc3Rpbmcgbm9kZSBzdGF0dXMuIFJlc3RhcnRpbmciCiAgICBlbGlmICEgb3V0cHV0PSQoY3VybCAtbSAiJHttYXhfc2Vjb25kc30iIC1mIC1zIC1TIGh0dHA6Ly8xMjcuMC4wLjE6MTAyNDgvaGVhbHRoeiAyPiYxKTsgdGhlbgogICAgICBmYWlsZWQ9dHJ1ZQogICAgICAjIFByaW50IHRoZSByZXNwb25zZSBhbmQvb3IgZXJyb3JzLgogICAgICBlY2hvICIkb3V0cHV0IgogICAgZmkKCiAgICBpZiBbWyAiJGZhaWxlZCIgPT0gInRydWUiIF1dOyB0aGVuCiAgICAgIGVjaG8gIkt1YmVsZXQgaXMgdW5oZWFsdGh5ISIKICAgICAgc3lzdGVtY3RsIGtpbGwga3ViZWxldAogICAgICAjIFdhaXQgZm9yIGEgd2hpbGUsIGFzIHdlIGRvbid0IHdhbnQgdG8ga2lsbCBpdCBhZ2FpbiBiZWZvcmUgaXQgaXMgcmVhbGx5IHVwLgogICAgICBzbGVlcCA2MAogICAgZWxzZQogICAgICBzbGVlcCAiJHtTTEVFUF9TRUNPTkRTfSIKICAgIGZpCiAgZG9uZQp9CgojIyMjIyMjIyMjIyMjIyBNYWluIEZ1bmN0aW9uICMjIyMjIyMjIyMjIyMjIyMKaWYgW1sgIiQjIiAtbmUgMSBdXTsgdGhlbgogIGVjaG8gIlVzYWdlOiBoZWFsdGgtbW9uaXRvci5zaCA8Y29udGFpbmVyLXJ1bnRpbWUva3ViZWxldD4iCiAgZXhpdCAxCmZpCgpTTEVFUF9TRUNPTkRTPTEwCmNvbXBvbmVudD0kMQplY2hvICJTdGFydCBrdWJlcm5ldGVzIGhlYWx0aCBtb25pdG9yaW5nIGZvciAke2NvbXBvbmVudH0iCmlmIFtbICIke2NvbXBvbmVudH0iID09ICJjb250YWluZXItcnVudGltZSIgXV07IHRoZW4KICBjb250YWluZXJfcnVudGltZV9tb25pdG9yaW5nCmVsaWYgW1sgIiR7Y29tcG9uZW50fSIgPT0gImt1YmVsZXQiIF1dOyB0aGVuCiAga3ViZWxldF9tb25pdG9yaW5nCmVsc2UKICBlY2hvICJIZWFsdGggbW9uaXRvcmluZyBmb3IgY29tcG9uZW50ICR7Y29tcG9uZW50fSBpcyBub3Qgc3VwcG9ydGVkISIKZmkK

- path: '/etc/systemd/journald.conf.d/max_disk_use.conf'
  permissions: '0644'
  encoding: 'b64'
  content: |-
    [Journal]
    SystemMaxUse=5G

- path: '/opt/load-kernel-modules.sh'
  permissions: '0755'
  encoding: 'b64'
  content: |-
    #!/usr/bin/env bash
    set -euo pipefail
    
    modprobe ip_vs
    modprobe ip_vs_rr
    modprobe ip_vs_wrr
    modprobe ip_vs_sh
    
    if modinfo nf_conntrack_ipv4 &> /dev/null; then
    modprobe nf_conntrack_ipv4
    else
    modprobe nf_conntrack
    fi

- path: '/etc/sysctl.d/k8s.conf'
  permissions: '0644'
  encoding: 'b64'
  content: |-
    net.bridge.bridge-nf-call-ip6tables = 1
    net.bridge.bridge-nf-call-iptables = 1
    kernel.panic_on_oops = 1
    kernel.panic = 10
    net.ipv4.ip_forward = 1
    vm.overcommit_memory = 1
    fs.inotify.max_user_watches = 1048576
    fs.inotify.max_user_instances = 8192

- path: '/etc/default/grub.d/60-swap-accounting.cfg'
  permissions: '0644'
  encoding: 'b64'
  content: |-
    # Added by kubermatic machine-controller
    # Enable cgroups memory and swap accounting
    GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1"

- path: '/opt/bin/setup'
  permissions: '0755'
  encoding: 'b64'
  content: |-
    #!/bin/bash
    set -xeuo pipefail
    if systemctl is-active ufw; then systemctl stop ufw; fi
    systemctl mask ufw
    systemctl restart systemd-modules-load.service
    sysctl --system
    
    # Override hostname if /etc/machine-name exists
    if [ -x "$(command -v hostnamectl)" ] && [ -s /etc/machine-name ]; then
    machine_name=$(cat /etc/machine-name)
    hostnamectl set-hostname ${machine_name}
    fi
    
    apt-get update
    
    DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \
    curl \
    ca-certificates \
    ceph-common \
    cifs-utils \
    conntrack \
    e2fsprogs \
    ebtables \
    ethtool \
    glusterfs-client \
    iptables \
    jq \
    kmod \
    openssh-client \
    nfs-common \
    socat \
    util-linux \
    ipvsadm
    apt-get update
    apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release
    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
    add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
    
    apt-get install -y --allow-downgrades containerd.io=1.6*
    apt-mark hold containerd.io
    
    systemctl daemon-reload
    systemctl enable --now containerd
    
    opt_bin=/opt/bin
    usr_local_bin=/usr/local/bin
    cni_bin_dir=/opt/cni/bin
    mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir"
    arch=${HOST_ARCH-}
    if [ -z "$arch" ]
    then
    case $(uname -m) in
    x86_64)
    arch="amd64"
    ;;
    aarch64)
    arch="arm64"
    ;;
    *)
    echo "unsupported CPU architecture, exiting"
    exit 1
    ;;
    esac
    fi
    CNI_VERSION="${CNI_VERSION:-v1.2.0}"
    cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION"
    cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz"
    curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename"
    cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256")
    cd "$cni_bin_dir"
    sha256sum -c <<<"$cni_sum"
    tar xvf "$cni_filename"
    rm -f "$cni_filename"
    cd -
    CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}"
    cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRI_TOOLS_RELEASE}"
    cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz"
    curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename"
    cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256")
    cri_tools_sum="$cri_tools_sum_value $cri_tools_filename"
    cd "$opt_bin"
    sha256sum -c <<<"$cri_tools_sum"
    tar xvf "$cri_tools_filename"
    rm -f "$cri_tools_filename"
    ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped"
    cd -
    KUBE_VERSION="${KUBE_VERSION:-v1.25.9}"
    kube_dir="$opt_bin/kubernetes-$KUBE_VERSION"
    kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch"
    kube_sum_file="$kube_dir/sha256"
    mkdir -p "$kube_dir"
    : >"$kube_sum_file"
  
    for bin in kubelet kubeadm kubectl; do
    curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin"
    chmod +x "$kube_dir/$bin"
    sum=$(curl -Lf "$kube_base_url/$bin.sha256")
    echo "$sum  $kube_dir/$bin" >>"$kube_sum_file"
    done
    sha256sum -c "$kube_sum_file"
    
    for bin in kubelet kubeadm kubectl; do
    ln -sf "$kube_dir/$bin" "$opt_bin"/$bin
    done
    
    # set kubelet nodeip environment variable
    /opt/bin/setup_net_env.sh
    curl -s -k -v --header 'Authorization: Bearer ${K8S_CLOUD_INIT_TOKEN}' ${K8S_API_ADDR}/api/v1/namespaces/cloud-init-settings/secrets/kubelet-bootstrap-config | jq '.data["kubeconfig"]' -r| base64 -d > /etc/kubernetes/bootstrap-kubelet.conf
    
    systemctl enable --now kubelet
    systemctl enable --now --no-block kubelet-healthcheck.service
    systemctl disable setup.service

- path: '/etc/systemd/system/kubelet.service'
  permissions: '0644'
  encoding: 'b64'
  content: |-
    [Unit]
    After=containerd.service
    Requires=containerd.service
    
    Description=kubelet: The Kubernetes Node Agent
    Documentation=https://kubernetes.io/docs/home/
    
    [Service]
    User=root
    Restart=always
    StartLimitInterval=0
    RestartSec=10
    CPUAccounting=true
    MemoryAccounting=true
    
    Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/"
    EnvironmentFile=-/etc/environment
    
    ExecStartPre=/bin/bash /opt/disable-swap.sh
    ExecStartPre=/bin/bash /opt/load-kernel-modules.sh
    ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh
    ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \
    --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \
    --kubeconfig=/var/lib/kubelet/kubeconfig \
    --config=/etc/kubernetes/kubelet.conf \
    --cert-dir=/etc/kubernetes/pki \
    --hostname-override=${KUBELET_HOSTNAME} \
    --exit-on-lock-contention \
    --lock-file=/tmp/kubelet.lock \
    --container-runtime=remote \
    --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
    --node-ip ${KUBELET_NODE_IP}
    
    [Install]
    WantedBy=multi-user.target

- path: '/etc/systemd/system/kubelet.service.d/extras.conf'
  permissions: '0644'
  encoding: 'b64'
  content: |-
    [Service]
    Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf"

- path: '/opt/bin/setup_net_env.sh'
  permissions: '0755'
  encoding: 'b64'
  content: |-
    #!/usr/bin/env bash
    echodate() {
      echo "[$(date -Is)]" "$@"
    }

    # get the default interface IP address
    DEFAULT_IFC_IP=$(ip -o  route get 1 | grep -oP "src \K\S+")
    
    if [ -z "${DEFAULT_IFC_IP}" ]
    then
      echodate "Failed to get IP address for the default route interface"
      exit 1
    fi
    
    # get the full hostname
    FULL_HOSTNAME=$(hostname -f)
    # if /etc/machine-name is not empty then use the hostname from there
    if [ -s /etc/machine-name ]; then
      FULL_HOSTNAME=$(cat /etc/machine-name)
    fi
    
    # write the nodeip_env file
    # we need the line below because flatcar has the same string "coreos" in that file
    if grep -q coreos /etc/os-release
    then
      echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf
    elif [ ! -d /etc/systemd/system/kubelet.service.d ]
    then
      echodate "Can't find kubelet service extras directory"
    exit 1
    else
      echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf
    fi

- path: '/etc/kubernetes/pki/ca.crt'
  permissions: '0644'
  encoding: 'b64'
  content: |-
    ${K8S_CA}

- path: '/etc/systemd/system/setup.service'
  permissions: '0644'
  encoding: 'b64'
  content: |-
    [Install]
    WantedBy=multi-user.target
    
    [Unit]
    Requires=network-online.target
    After=network-online.target
    
    [Service]
    Type=oneshot
    RemainAfterExit=true
    EnvironmentFile=-/etc/environment
    ExecStart=/opt/bin/supervise.sh /opt/bin/setup

- path: '/etc/profile.d/opt-bin-path.sh'
  permissions: '0644'
  encoding: 'b64'
  content: |-
    export PATH="/opt/bin:$PATH"

- path: '/etc/kubernetes/kubelet.conf'
  permissions: '0644'
  encoding: 'b64'
  content: |-
    apiVersion: kubelet.config.k8s.io/v1beta1
    kind: KubeletConfiguration
    authentication:
      anonymous:
        enabled: false
      webhook:
        cacheTTL: 0s
        enabled: true
      x509:
        clientCAFile: /etc/kubernetes/pki/ca.crt
    authorization:
      mode: Webhook
      webhook:
        cacheAuthorizedTTL: 0s
        cacheUnauthorizedTTL: 0s
    cgroupDriver: systemd
    clusterDNS:
      - "169.254.20.10"
    clusterDomain: cluster.local
    containerLogMaxSize: 100Mi
    containerLogMaxFiles: 5
    cpuManagerReconcilePeriod: 0s
    evictionPressureTransitionPeriod: 0s
    featureGates:
      RotateKubeletServerCertificate: true
    fileCheckFrequency: 0s
    httpCheckFrequency: 0s
    imageMinimumGCAge: 0s
    logging:
      flushFrequency: 0
      options:
        json:
          infoBufferSize: "0"
      verbosity: 0
    memorySwap: {}
    nodeStatusReportFrequency: 0s
    nodeStatusUpdateFrequency: 0s
    protectKernelDefaults: true
    readOnlyPort: 0
    rotateCertificates: true
    runtimeRequestTimeout: 0s
    serverTLSBootstrap: true
    shutdownGracePeriod: 0s
    shutdownGracePeriodCriticalPods: 0s
    staticPodPath: /etc/kubernetes/manifests
    streamingConnectionIdleTimeout: 0s
    syncFrequency: 0s
    kubeReserved:
      cpu: 200m
      ephemeral-storage: 1Gi
      memory: 200Mi
    systemReserved:
      cpu: 200m
      ephemeral-storage: 1Gi
      memory: 200Mi
    evictionHard:
      imagefs.available: 15%
      memory.available: 100Mi
      nodefs.available: 10%
      nodefs.inodesFree: 5%
    tlsCipherSuites:
      - TLS_AES_128_GCM_SHA256
      - TLS_AES_256_GCM_SHA384
      - TLS_CHACHA20_POLY1305_SHA256
      - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
      - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
      - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
      - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
      - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
      - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305
    volumePluginDir: /var/lib/kubelet/volumeplugins
    volumeStatsAggPeriod: 0s

- path: '/etc/systemd/system/kubelet-healthcheck.service'
  permissions: '0644'
  encoding: 'b64'
  content: |-
    [Unit]
    Requires=kubelet.service
    After=kubelet.service
    
    [Service]
    EnvironmentFile=-/etc/environment
    ExecStart=/opt/bin/health-monitor.sh kubelet
    
    [Install]
    WantedBy=multi-user.target

- path: '/opt/disable-swap.sh'
  permissions: '0755'
  encoding: 'b64'
  content: |-
    #!/usr/bin/env bash
    set -euo pipefail
    
    # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud
    # providers swap gets enabled on reboot or after the setup script has finished executing.
    sed -i.orig '/.*swap.*/d' /etc/fstab
    swapoff -a

- path: '/etc/systemd/system/containerd.service.d/environment.conf'
  permissions: '0644'
  encoding: 'b64'
  content: |-
    [Service]
    Restart=always
    EnvironmentFile=-/etc/environment

- path: '/etc/crictl.yaml'
  permissions: '0644'
  encoding: 'b64'
  content: |-
    runtime-endpoint: unix:///run/containerd/containerd.sock

- path: '/etc/containerd/config.toml'
  permissions: '0600'
  encoding: 'b64'
  content: |-
    version = 2
    
    [metrics]
    address = "127.0.0.1:1338"
    
    [plugins]
    [plugins."io.containerd.grpc.v1.cri"]
    [plugins."io.containerd.grpc.v1.cri".containerd]
    [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
    [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
    runtime_type = "io.containerd.runc.v2"
    [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
    SystemdCgroup = true
    [plugins."io.containerd.grpc.v1.cri".registry]
    [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
    [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
    endpoint = ["https://registry-1.docker.io"]