feat: define resource reservation

Set memory/cpu resource reservation for system processes.
It helps system processes to allocate memory on memory pressure
situation.

Signed-off-by: Serge Logvinov <[email protected]>
Signed-off-by: Andrey Smirnov <[email protected]>

go.mod

@@ -35,7 +35,7 @@ require (
 	github.com/aws/aws-sdk-go v1.43.8
 	github.com/beevik/ntp v0.3.0
 	github.com/cenkalti/backoff/v4 v4.1.2
-	github.com/containerd/cgroups v1.0.3
+	github.com/containerd/cgroups v1.0.4-0.20220301195952-2e502f6b9e43
 	github.com/containerd/containerd v1.6.0
 	github.com/containerd/cri v1.19.0
 	github.com/containerd/typeurl v1.0.2

go.sum

@@ -249,8 +249,9 @@ github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4S
 github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo=
 github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE=
 github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU=
-github.com/containerd/cgroups v1.0.3 h1:ADZftAkglvCiD44c77s5YmMqaP2pzVCFZvBmAlBdAP4=
 github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8=
+github.com/containerd/cgroups v1.0.4-0.20220301195952-2e502f6b9e43 h1:heo8yArk63uJ8TyN/fZ3Tj1xlAPfT8GaXTqzHH+t07w=
+github.com/containerd/cgroups v1.0.4-0.20220301195952-2e502f6b9e43/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8=
 github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
 github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
 github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE=

internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go

@@ -23,6 +23,7 @@ import (
 	"text/template"
 	"time"
 
+	"github.com/AlekSi/pointer"
 	"github.com/containerd/cgroups"
 	cgroupsv2 "github.com/containerd/cgroups/v2"
 	multierror "github.com/hashicorp/go-multierror"
@@ -151,32 +152,78 @@ func CreateSystemCgroups(seq runtime.Sequence, data interface{}) (runtime.TaskEx
 			}
 		}
 
-		groups := []string{
-			constants.CgroupInit,
-			constants.CgroupRuntime,
-			constants.CgroupPodRuntime,
-			constants.CgroupKubelet,
+		groups := []struct {
+			name      string
+			resources *cgroupsv2.Resources
+		}{
+			{
+				name: constants.CgroupInit,
+				resources: &cgroupsv2.Resources{
+					Memory: &cgroupsv2.Memory{
+						Min: pointer.ToInt64(constants.CgroupInitReservedMemory),
+						Low: pointer.ToInt64(constants.CgroupInitReservedMemory * 2),
+					},
+				},
+			},
+			{
+				name: constants.CgroupSystem,
+				resources: &cgroupsv2.Resources{
+					Memory: &cgroupsv2.Memory{
+						Min: pointer.ToInt64(constants.CgroupSystemReservedMemory),
+						Low: pointer.ToInt64(constants.CgroupSystemReservedMemory * 2),
+					},
+				},
+			},
+			{
+				name:      constants.CgroupSystemRuntime,
+				resources: &cgroupsv2.Resources{},
+			},
+			{
+				name: constants.CgroupPodRuntime,
+				resources: &cgroupsv2.Resources{
+					Memory: &cgroupsv2.Memory{
+						Min: pointer.ToInt64(constants.CgroupPodRuntimeReservedMemory),
+						Low: pointer.ToInt64(constants.CgroupPodRuntimeReservedMemory * 2),
+					},
+				},
+			},
+			{
+				name: constants.CgroupKubelet,
+				resources: &cgroupsv2.Resources{
+					Memory: &cgroupsv2.Memory{
+						Min: pointer.ToInt64(constants.CgroupKubeletReservedMemory),
+						Low: pointer.ToInt64(constants.CgroupKubeletReservedMemory * 2),
+					},
+				},
+			},
 		}
 
 		for _, c := range groups {
 			if cgroups.Mode() == cgroups.Unified {
-				cg, err := cgroupsv2.NewManager(constants.CgroupMountPath, c, &cgroupsv2.Resources{})
+				resources := c.resources
+
+				if r.State().Platform().Mode() == runtime.ModeContainer {
+					// don't attempt to set resources in container mode, as they might conflict with the parent cgroup tree
+					resources = &cgroupsv2.Resources{}
+				}
+
+				cg, err := cgroupsv2.NewManager(constants.CgroupMountPath, c.name, resources)
 				if err != nil {
 					return fmt.Errorf("failed to create cgroup: %w", err)
 				}
 
-				if c == constants.CgroupInit {
+				if c.name == constants.CgroupInit {
 					if err := cg.AddProc(uint64(os.Getpid())); err != nil {
 						return fmt.Errorf("failed to move init process to cgroup: %w", err)
 					}
 				}
 			} else {
-				cg, err := cgroups.New(cgroups.V1, cgroups.StaticPath(c), &specs.LinuxResources{})
+				cg, err := cgroups.New(cgroups.V1, cgroups.StaticPath(c.name), &specs.LinuxResources{})
 				if err != nil {
 					return fmt.Errorf("failed to create cgroup: %w", err)
 				}
 
-				if c == constants.CgroupInit {
+				if c.name == constants.CgroupInit {
 					if err := cg.Add(cgroups.Process{
 						Pid: os.Getpid(),
 					}); err != nil {

internal/app/machined/pkg/system/services/containerd.go

@@ -78,7 +78,7 @@ func (c *Containerd) Runner(r runtime.Runtime) (runner.Runner, error) {
 		runner.WithLoggingManager(r.Logging()),
 		runner.WithEnv(env),
 		runner.WithOOMScoreAdj(-999),
-		runner.WithCgroupPath(constants.CgroupRuntime),
+		runner.WithCgroupPath(constants.CgroupSystemRuntime),
 	),
 		restart.WithType(restart.Forever),
 	), nil

internal/app/machined/pkg/system/services/udevd.go

@@ -79,7 +79,7 @@ func (c *Udevd) Runner(r runtime.Runtime) (runner.Runner, error) {
 		args,
 		runner.WithLoggingManager(r.Logging()),
 		runner.WithEnv(env),
-		runner.WithCgroupPath(constants.CgroupRuntime),
+		runner.WithCgroupPath(constants.CgroupSystemRuntime),
 	),
 		restart.WithType(restart.Forever),
 	), nil

pkg/machinery/constants/constants.go

@@ -287,7 +287,7 @@ const (
 	KubeletSystemReservedCPU = "50m"
 
 	// KubeletSystemReservedMemory memory system reservation value for kubelet kubeconfig.
-	KubeletSystemReservedMemory = "128Mi"
+	KubeletSystemReservedMemory = "192Mi"
 
 	// KubeletSystemReservedPid pid system reservation value for kubelet kubeconfig.
 	KubeletSystemReservedPid = "100"
@@ -458,21 +458,33 @@ const (
 	// CgroupInit is the cgroup name for init process.
 	CgroupInit = "/init"
 
+	// CgroupInitReservedMemory is the hard memory protection for the init process.
+	CgroupInitReservedMemory = 96 * 1024 * 1024
+
 	// CgroupSystem is the cgroup name for system processes.
 	CgroupSystem = "/system"
 
-	// CgroupRuntime is the cgroup name for containerd runtime processes.
-	CgroupRuntime = CgroupSystem + "/runtime"
+	// CgroupSystemReservedMemory is the hard memory protection for the system processes.
+	CgroupSystemReservedMemory = 96 * 1024 * 1024
+
+	// CgroupSystemRuntime is the cgroup name for containerd runtime processes.
+	CgroupSystemRuntime = CgroupSystem + "/runtime"
 
 	// CgroupExtensions is the cgroup name for system extension processes.
 	CgroupExtensions = CgroupSystem + "/extensions"
 
 	// CgroupPodRuntime is the cgroup name for kubernetes containerd runtime processes.
 	CgroupPodRuntime = "/podruntime/runtime"
 
+	// CgroupPodRuntimeReservedMemory is the hard memory protection for the cri runtime processes.
+	CgroupPodRuntimeReservedMemory = 128 * 1024 * 1024
+
 	// CgroupKubelet is the cgroup name for kubelet process.
 	CgroupKubelet = "/podruntime/kubelet"
 
+	// CgroupKubeletReservedMemory is the hard memory protection for the kubelet processes.
+	CgroupKubeletReservedMemory = 64 * 1024 * 1024
+
 	// FlannelCNI is the string to use Tanos-managed Flannel CNI (default).
 	FlannelCNI = "flannel"