feat: store audit logs to disk

Instead of bundling the apiserver audit logs with the rest of the apiserver logs, we should store them separately to file, assuring reasonable defaults for retention and rotation. Fixes #5000 Signed-off-by: Seán C McCord <[email protected]>
siderolabs · Feb 21, 2022 · 4d419a0 · 4d419a0
1 parent 8ef3d85
commit 4d419a0
Show file tree

Hide file tree

Showing 5 changed files with 59 additions and 9 deletions.
diff --git a/hack/release.toml b/hack/release.toml
@@ -36,6 +36,14 @@ A number of conformance tweaks have been made to the `kubelet` to allow it to ru
 This includes both kubelet configuration options and sysctls.
 Of particular note is that Talos now sets the `kernel.panic` reboot interval to 10s instead of 1s.
 If your kubelet fails to start after the upgrade, please check the `kubelet` logs to determine the problem.
+"""
+
+    [notes.auditlog]
+        title = "API Server audit logs"
+        description="""\
+`kube-apiserver` is now configured to store its audit logs separately from the `kube-apiserver` standard logs and directly to file.
+The `kube-apiserver` will maintain the rotation and retirement of these logs, which are stored in `/var/log/audit/`.
+Previously, the audit logs were sent to `kube-apiserver`'s `stdout`, along with the rest of its logs, to be collected in the usual manner by Kubernetes.
 """
 
     [notes.updates]

diff --git a/internal/app/machined/pkg/controllers/k8s/control_plane_static_pod.go b/internal/app/machined/pkg/controllers/k8s/control_plane_static_pod.go
@@ -262,10 +262,10 @@ func (ctrl *ControlPlaneStaticPodController) manageAPIServer(ctx context.Context
 		"tls-cipher-suites":                "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256", //nolint:lll
 		"encryption-provider-config":       filepath.Join(constants.KubernetesAPIServerSecretsDir, "encryptionconfig.yaml"),
 		"audit-policy-file":                filepath.Join(constants.KubernetesAPIServerSecretsDir, "auditpolicy.yaml"),
-		"audit-log-path":                   "-",
+		"audit-log-path":                   filepath.Join(constants.KubernetesAuditLogDir, "kube-apiserver.log"),
 		"audit-log-maxage":                 "30",
-		"audit-log-maxbackup":              "3",
-		"audit-log-maxsize":                "50",
+		"audit-log-maxbackup":              "10",
+		"audit-log-maxsize":                "100",
 		"profiling":                        "false",
 		"etcd-cafile":                      filepath.Join(constants.KubernetesAPIServerSecretsDir, "etcd-client-ca.crt"),
 		"etcd-certfile":                    filepath.Join(constants.KubernetesAPIServerSecretsDir, "etcd-client.crt"),
@@ -356,6 +356,11 @@ func (ctrl *ControlPlaneStaticPodController) manageAPIServer(ctx context.Context
 								MountPath: constants.KubernetesAPIServerSecretsDir,
 								ReadOnly:  true,
 							},
+							{
+								Name:      "audit",
+								MountPath: constants.KubernetesAuditLogDir,
+								ReadOnly:  false,
+							},
 						}, volumeMounts(cfg.ExtraVolumes)...),
 						Resources: v1.ResourceRequirements{
 							Requests: v1.ResourceList{
@@ -379,6 +384,14 @@ func (ctrl *ControlPlaneStaticPodController) manageAPIServer(ctx context.Context
 							},
 						},
 					},
+					{
+						Name: "audit",
+						VolumeSource: v1.VolumeSource{
+							HostPath: &v1.HostPathVolumeSource{
+								Path: constants.KubernetesAuditLogDir,
+							},
+						},
+					},
 				}, volumes(cfg.ExtraVolumes)...),
 			},
 		})

diff --git a/internal/app/machined/pkg/controllers/k8s/control_plane_static_pod_test.go b/internal/app/machined/pkg/controllers/k8s/control_plane_static_pod_test.go
@@ -178,8 +178,8 @@ func (suite *ControlPlaneStaticPodSuite) TestReconcileExtraMounts() {
 	apiServerPod, err := k8sadapter.StaticPod(r.(*k8s.StaticPod)).Pod()
 	suite.Require().NoError(err)
 
-	suite.Assert().Len(apiServerPod.Spec.Volumes, 2)
-	suite.Assert().Len(apiServerPod.Spec.Containers[0].VolumeMounts, 2)
+	suite.Assert().Len(apiServerPod.Spec.Volumes, 3)
+	suite.Assert().Len(apiServerPod.Spec.Containers[0].VolumeMounts, 3)
 
 	suite.Assert().Equal(v1.Volume{
 		Name: "secrets",
@@ -190,26 +190,41 @@ func (suite *ControlPlaneStaticPodSuite) TestReconcileExtraMounts() {
 		},
 	}, apiServerPod.Spec.Volumes[0])
 
+	suite.Assert().Equal(v1.Volume{
+		Name: "audit",
+		VolumeSource: v1.VolumeSource{
+			HostPath: &v1.HostPathVolumeSource{
+				Path: constants.KubernetesAuditLogDir,
+			},
+		},
+	}, apiServerPod.Spec.Volumes[1])
+
 	suite.Assert().Equal(v1.Volume{
 		Name: "foo",
 		VolumeSource: v1.VolumeSource{
 			HostPath: &v1.HostPathVolumeSource{
 				Path: "/var/lib",
 			},
 		},
-	}, apiServerPod.Spec.Volumes[1])
+	}, apiServerPod.Spec.Volumes[2])
 
 	suite.Assert().Equal(v1.VolumeMount{
 		Name:      "secrets",
 		MountPath: constants.KubernetesAPIServerSecretsDir,
 		ReadOnly:  true,
 	}, apiServerPod.Spec.Containers[0].VolumeMounts[0])
 
+	suite.Assert().Equal(v1.VolumeMount{
+		Name:      "audit",
+		MountPath: constants.KubernetesAuditLogDir,
+		ReadOnly:  false,
+	}, apiServerPod.Spec.Containers[0].VolumeMounts[1])
+
 	suite.Assert().Equal(v1.VolumeMount{
 		Name:      "foo",
 		MountPath: "/var/foo",
 		ReadOnly:  true,
-	}, apiServerPod.Spec.Containers[0].VolumeMounts[1])
+	}, apiServerPod.Spec.Containers[0].VolumeMounts[2])
 }
 
 func (suite *ControlPlaneStaticPodSuite) TestReconcileExtraArgs() {

diff --git a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go
@@ -713,12 +713,23 @@ func SetupSharedFilesystems(seq runtime.Sequence, data interface{}) (runtime.Tas
 // SetupVarDirectory represents the SetupVarDirectory task.
 func SetupVarDirectory(seq runtime.Sequence, data interface{}) (runtime.TaskExecutionFunc, string) {
 	return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) {
-		for _, p := range []string{"/var/log/containers", "/var/log/pods", "/var/lib/kubelet", "/var/run/lock"} {
+		for _, p := range []string{"/var/log/audit", "/var/log/containers", "/var/log/pods", "/var/lib/kubelet", "/var/run/lock"} {
 			if err = os.MkdirAll(p, 0o700); err != nil {
 				return err
 			}
 		}
 
+		// Handle Kubernetes directories which need different ownership
+		for _, p := range []string{constants.KubernetesAuditLogDir} {
+			if err = os.MkdirAll(p, 0o700); err != nil {
+				return err
+			}
+
+			if err = os.Chown(p, constants.KubernetesRunUser, -1); err != nil {
+				return fmt.Errorf("failed to chown %s: %w", p, err)
+			}
+		}
+
 		return nil
 	}, "setupVarDirectory"
 }

diff --git a/pkg/machinery/constants/constants.go b/pkg/machinery/constants/constants.go
@@ -201,7 +201,10 @@ const (
 	// KubebernetesStaticSecretsDir defines ephemeral directory which contains rendered secrets for controlplane components.
 	KubebernetesStaticSecretsDir = "/system/secrets/kubernetes"
 
-	// KubernetesAPIServerSecretsDir defines ephemeral directory with kube-apiserver secrets.
+	// KubernetesAuditLogDir defines the ephemeral directory where the kube-apiserver will store its audit logs.
+	KubernetesAuditLogDir = EphemeralMountPoint + "/" + "log" + "/" + "audit" + "/" + "kube"
+
+	// KubernetesAPIServerSecretsDir defines directory with kube-apiserver secrets.
 	KubernetesAPIServerSecretsDir = KubebernetesStaticSecretsDir + "/" + "kube-apiserver"
 
 	// KubernetesControllerManagerSecretsDir defines ephemeral directory with kube-controller-manager secrets.