feat: follow KEP-2568 non-root enhancements

KEP-2568: https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/kubeadm/2568-kubeadm-non-root-control-plane Deviation: - example sets UID/GID in container context, its safer to do this in pod context Signed-off-by: Nico Berlee <[email protected]> Signed-off-by: Andrey Smirnov <[email protected]>
siderolabs · Jul 18, 2022 · be98cb8 · be98cb8
1 parent 87ea1d9
commit be98cb8
Show file tree

Hide file tree

Showing 5 changed files with 71 additions and 10 deletions.
diff --git a/internal/app/machined/pkg/controllers/k8s/control_plane_static_pod.go b/internal/app/machined/pkg/controllers/k8s/control_plane_static_pod.go
@@ -423,12 +423,26 @@ func (ctrl *ControlPlaneStaticPodController) manageAPIServer(ctx context.Context
 								v1.ResourceMemory: apiresource.MustParse("512Mi"),
 							},
 						},
+						SecurityContext: &v1.SecurityContext{
+							AllowPrivilegeEscalation: pointer.To(false),
+							Capabilities: &v1.Capabilities{
+								Drop: []v1.Capability{"ALL"},
+								// kube-apiserver binary has cap_net_bind_service=+ep set.
+								// It does not matter if ports < 1024 are configured, the setcap flag causes a capability dependency.
+								// https://github.com/kubernetes/kubernetes/blob/5b92e46b2238b4d84358451013e634361084ff7d/build/server-image/kube-apiserver/Dockerfile#L26
+								Add: []v1.Capability{"NET_BIND_SERVICE"},
+							},
+							SeccompProfile: &v1.SeccompProfile{
+								Type: v1.SeccompProfileTypeRuntimeDefault,
+							},
+						},
 					},
 				},
 				HostNetwork: true,
 				SecurityContext: &v1.PodSecurityContext{
 					RunAsNonRoot: pointer.To(true),
-					RunAsUser:    pointer.To[int64](constants.KubernetesRunUser),
+					RunAsUser:    pointer.To[int64](constants.KubernetesAPIServerRunUser),
+					RunAsGroup:   pointer.To[int64](constants.KubernetesAPIServerRunGroup),
 				},
 				Volumes: append([]v1.Volume{
 					{
@@ -568,12 +582,22 @@ func (ctrl *ControlPlaneStaticPodController) manageControllerManager(ctx context
 								v1.ResourceMemory: apiresource.MustParse("256Mi"),
 							},
 						},
+						SecurityContext: &v1.SecurityContext{
+							AllowPrivilegeEscalation: pointer.To(false),
+							Capabilities: &v1.Capabilities{
+								Drop: []v1.Capability{"ALL"},
+							},
+							SeccompProfile: &v1.SeccompProfile{
+								Type: v1.SeccompProfileTypeRuntimeDefault,
+							},
+						},
 					},
 				},
 				HostNetwork: true,
 				SecurityContext: &v1.PodSecurityContext{
 					RunAsNonRoot: pointer.To(true),
-					RunAsUser:    pointer.To[int64](constants.KubernetesRunUser),
+					RunAsUser:    pointer.To[int64](constants.KubernetesControllerManagerRunUser),
+					RunAsGroup:   pointer.To[int64](constants.KubernetesControllerManagerRunGroup),
 				},
 				Volumes: append([]v1.Volume{
 					{
@@ -678,12 +702,22 @@ func (ctrl *ControlPlaneStaticPodController) manageScheduler(ctx context.Context
 								v1.ResourceMemory: apiresource.MustParse("64Mi"),
 							},
 						},
+						SecurityContext: &v1.SecurityContext{
+							AllowPrivilegeEscalation: pointer.To(false),
+							Capabilities: &v1.Capabilities{
+								Drop: []v1.Capability{"ALL"},
+							},
+							SeccompProfile: &v1.SeccompProfile{
+								Type: v1.SeccompProfileTypeRuntimeDefault,
+							},
+						},
 					},
 				},
 				HostNetwork: true,
 				SecurityContext: &v1.PodSecurityContext{
 					RunAsNonRoot: pointer.To(true),
-					RunAsUser:    pointer.To[int64](constants.KubernetesRunUser),
+					RunAsUser:    pointer.To[int64](constants.KubernetesSchedulerRunUser),
+					RunAsGroup:   pointer.To[int64](constants.KubernetesSchedulerRunGroup),
 				},
 				Volumes: append([]v1.Volume{
 					{

diff --git a/internal/app/machined/pkg/controllers/k8s/render_config_static_pods.go b/internal/app/machined/pkg/controllers/k8s/render_config_static_pods.go
@@ -93,11 +93,15 @@ func (ctrl *RenderConfigsStaticPodController) Run(ctx context.Context, r control
 		for _, pod := range []struct {
 			name      string
 			directory string
+			uid       int
+			gid       int
 			configs   []configFile
 		}{
 			{
 				name:      "kube-apiserver",
 				directory: constants.KubernetesAPIServerConfigDir,
+				uid:       constants.KubernetesAPIServerRunUser,
+				gid:       constants.KubernetesAPIServerRunGroup,
 				configs: []configFile{
 					{
 						filename: "admission-control-config.yaml",
@@ -128,7 +132,7 @@ func (ctrl *RenderConfigsStaticPodController) Run(ctx context.Context, r control
 					return fmt.Errorf("error writing configuration %q for %q: %w", configFile.filename, pod.name, err)
 				}
 
-				if err = os.Chown(filepath.Join(pod.directory, configFile.filename), constants.KubernetesRunUser, -1); err != nil {
+				if err = os.Chown(filepath.Join(pod.directory, configFile.filename), pod.uid, pod.gid); err != nil {
 					return fmt.Errorf("error chowning %q for %q: %w", configFile.filename, pod.name, err)
 				}
 			}

diff --git a/internal/app/machined/pkg/controllers/k8s/render_secrets_static_pod.go b/internal/app/machined/pkg/controllers/k8s/render_secrets_static_pod.go
@@ -144,12 +144,16 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control
 		for _, pod := range []struct {
 			name      string
 			directory string
+			uid       int
+			gid       int
 			secrets   []secret
 			templates []template
 		}{
 			{
 				name:      "kube-apiserver",
 				directory: constants.KubernetesAPIServerSecretsDir,
+				uid:       constants.KubernetesAPIServerRunUser,
+				gid:       constants.KubernetesAPIServerRunGroup,
 				secrets: []secret{
 					{
 						getter:       func() *x509.PEMEncodedCertificateAndKey { return rootEtcdSecrets.EtcdCA },
@@ -208,6 +212,8 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control
 			{
 				name:      "kube-controller-manager",
 				directory: constants.KubernetesControllerManagerSecretsDir,
+				uid:       constants.KubernetesControllerManagerRunUser,
+				gid:       constants.KubernetesControllerManagerRunGroup,
 				secrets: []secret{
 					{
 						getter:       func() *x509.PEMEncodedCertificateAndKey { return rootK8sSecrets.CA },
@@ -234,6 +240,8 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control
 			{
 				name:      "kube-scheduler",
 				directory: constants.KubernetesSchedulerSecretsDir,
+				uid:       constants.KubernetesSchedulerRunUser,
+				gid:       constants.KubernetesSchedulerRunGroup,
 				templates: []template{
 					{
 						filename: "kubeconfig",
@@ -254,7 +262,7 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control
 						return fmt.Errorf("error writing certificate %q for %q: %w", secret.certFilename, pod.name, err)
 					}
 
-					if err = os.Chown(filepath.Join(pod.directory, secret.certFilename), constants.KubernetesRunUser, -1); err != nil {
+					if err = os.Chown(filepath.Join(pod.directory, secret.certFilename), pod.uid, pod.gid); err != nil {
 						return fmt.Errorf("error chowning %q for %q: %w", secret.certFilename, pod.name, err)
 					}
 				}
@@ -264,7 +272,7 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control
 						return fmt.Errorf("error writing key %q for %q: %w", secret.keyFilename, pod.name, err)
 					}
 
-					if err = os.Chown(filepath.Join(pod.directory, secret.keyFilename), constants.KubernetesRunUser, -1); err != nil {
+					if err = os.Chown(filepath.Join(pod.directory, secret.keyFilename), pod.uid, pod.gid); err != nil {
 						return fmt.Errorf("error chowning %q for %q: %w", secret.keyFilename, pod.name, err)
 					}
 				}
@@ -298,7 +306,7 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control
 					return fmt.Errorf("error writing template %q for %q: %w", templ.filename, pod.name, err)
 				}
 
-				if err = os.Chown(filepath.Join(pod.directory, templ.filename), constants.KubernetesRunUser, -1); err != nil {
+				if err = os.Chown(filepath.Join(pod.directory, templ.filename), pod.uid, pod.gid); err != nil {
 					return fmt.Errorf("error chowning %q for %q: %w", templ.filename, pod.name, err)
 				}
 			}

diff --git a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go
@@ -856,7 +856,7 @@ func SetupVarDirectory(seq runtime.Sequence, data interface{}) (runtime.TaskExec
 				return err
 			}
 
-			if err = os.Chown(p, constants.KubernetesRunUser, -1); err != nil {
+			if err = os.Chown(p, constants.KubernetesAPIServerRunUser, constants.KubernetesAPIServerRunGroup); err != nil {
 				return fmt.Errorf("failed to chown %s: %w", p, err)
 			}
 		}

diff --git a/pkg/machinery/constants/constants.go b/pkg/machinery/constants/constants.go
@@ -232,8 +232,23 @@ const (
 	// KubernetesSchedulerSecretsDir defines ephemeral directory with kube-scheduler secrets.
 	KubernetesSchedulerSecretsDir = KubebernetesStaticSecretsDir + "/" + "kube-scheduler"
 
-	// KubernetesRunUser defines UID to run control plane components.
-	KubernetesRunUser = 65534
+	// KubernetesAPIServerRunUser defines UID to the API Server.
+	KubernetesAPIServerRunUser = 65534
+
+	// KubernetesAPIServerRunGroup defines GID to run the API Server.
+	KubernetesAPIServerRunGroup = 65534
+
+	// KubernetesControllerManagerRunUser defines UID to the Controller Manager.
+	KubernetesControllerManagerRunUser = 65535
+
+	// KubernetesControllerManagerRunGroup defines GID to run the Controller Manager.
+	KubernetesControllerManagerRunGroup = 65535
+
+	// KubernetesSchedulerRunUser defines UID to the Scheduler.
+	KubernetesSchedulerRunUser = 65536
+
+	// KubernetesSchedulerRunGroup defines GID to run the Scheduler.
+	KubernetesSchedulerRunGroup = 65536
 
 	// KubeletBootstrapKubeconfig is the path to the kubeconfig required to
 	// bootstrap the kubelet.