Fix Kubelet and Containerd when using cgroupfs as cgroup driver (#8123)

docs/vars.md

@@ -119,7 +119,7 @@ Stack](https://github.com/kubernetes-sigs/kubespray/blob/master/docs/dns-stack.m
   ``--insecure-registry=myregistry.mydomain:5000``
 * *docker_plugins* - This list can be used to define [Docker plugins](https://docs.docker.com/engine/extend/) to install.
 * *containerd_default_runtime* - Sets the default Containerd runtime used by the Kubernetes CRI plugin.
-* *containerd_runtimes* - Sets the Containerd runtime attributes used by the Kubernetes CRI plugin.
+* *containerd_additional_runtimes* - Sets the additional Containerd runtimes used by the Kubernetes CRI plugin.
   [Default config](https://github.com/kubernetes-sigs/kubespray/blob/master/roles/container-engine/containerd/defaults/main.yml) can be overriden in inventory vars.
 * *http_proxy/https_proxy/no_proxy/no_proxy_exclude_workers/additional_no_proxy* - Proxy variables for deploying behind a
   proxy. Note that no_proxy defaults to all internal cluster IPs and hostnames

roles/container-engine/containerd/defaults/main.yml

@@ -7,13 +7,15 @@ containerd_oom_score: 0
 containerd_default_runtime: "runc"
 # containerd_snapshotter: "native"
 
-containerd_runtimes:
-  - name: runc
-    type: "io.containerd.runc.v2"
-    engine: ""
-    root: ""
-    options:
-      systemdCgroup: "true"
+containerd_runc_runtime:
+  name: runc
+  type: "io.containerd.runc.v2"
+  engine: ""
+  root: ""
+  options:
+    systemCgroup: "true"
+
+containerd_additional_runtimes: []
 # Example for Kata Containers as additional runtime:
 #  - name: kata
 #    type: "io.containerd.kata.v2"

roles/container-engine/containerd/tasks/facts.yml

@@ -0,0 +1,6 @@
+---
+
+- name: set kubelet_config_extra_args options when cgroupfs is used
+  set_fact:
+    containerd_runc_runtime: "{{ containerd_runc_runtime | combine({'options':{'systemCgroup':'false'}}) }}"
+  when: not containerd_use_systemd_cgroup

roles/container-engine/containerd/tasks/main.yml

@@ -1,4 +1,8 @@
 ---
+- import_tasks: facts.yml
+  tags:
+    - facts
+
 - name: Fail containerd setup if distribution is not supported
   fail:
     msg: "{{ ansible_distribution }} is not supported by containerd."

roles/container-engine/containerd/templates/config.toml.j2

@@ -22,7 +22,15 @@ oom_score = {{ containerd_oom_score }}
       default_runtime_name = "{{ containerd_default_runtime | default('runc') }}"
       snapshotter = "{{ containerd_snapshotter | default('overlayfs') }}"
       [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
-{% for runtime in containerd_runtimes %}
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ containerd_runc_runtime.name }}]
+          runtime_type = "{{ containerd_runc_runtime.type }}"
+          runtime_engine = "{{ containerd_runc_runtime.engine}}"
+          runtime_root = "{{ containerd_runc_runtime.root }}"
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ containerd_runc_runtime.name }}.options]
+{% for key, value in containerd_runc_runtime.options.items() %}
+            {{ key }} = {{ value }}
+{% endfor %}
+{% for runtime in containerd_additional_runtimes %}
         [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ runtime.name }}]
           runtime_type = "{{ runtime.type }}"
           runtime_engine = "{{ runtime.engine }}"

roles/kubernetes/node/defaults/main.yml

@@ -14,11 +14,14 @@ kube_resolv_conf: "/etc/resolv.conf"
 # Set to empty to avoid cgroup creation
 kubelet_enforce_node_allocatable: "\"\""
 
-# Set runtime cgroups
+# Set runtime and kubelet cgroups when using systemd as cgroup driver (default)
 kubelet_runtime_cgroups: "/systemd/system.slice"
-# Set kubelet cgroups
 kubelet_kubelet_cgroups: "/systemd/system.slice"
 
+# Set runtime and kubelet cgroups when using cgroupfs as cgroup driver
+kubelet_runtime_cgroups_cgroupfs: "/system.slice/containerd.service"
+kubelet_kubelet_cgroups_cgroupfs: "/system.slice/kubelet.slice"
+
 ### fail with swap on (default true)
 kubelet_fail_swap_on: true
 
@@ -66,6 +69,11 @@ kubelet_max_pods: 110
 ## Support parameters to be passed to kubelet via kubelet-config.yaml
 kubelet_config_extra_args: {}
 
+## Parameters to be passed to kubelet via kubelet-config.yaml when cgroupfs is used as cgroup driver
+kubelet_config_extra_args_cgroupfs:
+  systemCgroups: /system.slice
+  cgroupRoot: /
+
 ## Support parameters to be passed to kubelet via kubelet-config.yaml only on nodes, not masters
 kubelet_node_config_extra_args: {}
 

roles/kubernetes/node/tasks/facts.yml

@@ -27,6 +27,23 @@
       {%- if containerd_use_systemd_cgroup -%}systemd{%- else -%}cgroupfs{%- endif -%}
   when: container_manager == 'containerd'
 
+- name: set kubelet_cgroup_driver
+  set_fact:
+    kubelet_cgroup_driver: "{{ kubelet_cgroup_driver_detected }}"
+  when: kubelet_cgroup_driver is undefined
+
+- name: set kubelet_cgroups options when cgroupfs is used
+  set_fact:
+    kubelet_runtime_cgroups: "{{ kubelet_runtime_cgroups_cgroupfs }}"
+    kubelet_kubelet_cgroups: "{{ kubelet_kubelet_cgroups_cgroupfs }}"
+  when: kubelet_cgroup_driver == 'cgroupfs'
+
+- name: set kubelet_config_extra_args options when cgroupfs is used
+  vars:
+  set_fact:
+    kubelet_config_extra_args: "{{ kubelet_config_extra_args | combine(kubelet_config_extra_args_cgroupfs) }}"
+  when: kubelet_cgroup_driver == 'cgroupfs'
+
 - name: os specific vars
   include_vars: "{{ item }}"
   with_first_found:

roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2

@@ -23,7 +23,7 @@ enforceNodeAllocatable:
 {% endfor %}
 {% endif %}
 staticPodPath: {{ kube_manifest_dir }}
-cgroupDriver: {{ kubelet_cgroup_driver|default(kubelet_cgroup_driver_detected) }}
+cgroupDriver: {{ kubelet_cgroup_driver }}
 containerLogMaxFiles: {{ kubelet_logfiles_max_nr }}
 containerLogMaxSize: {{ kubelet_logfiles_max_size }}
 maxPods: {{ kubelet_max_pods }}