Add and update PodSecurityPolicy to match SecurityContext

This changeset add and update `PodSecurityPolicy` for open source and enterprise components to match the recent work for SecurityContext in [1]. This is also required for all components to work in a CIS hardened RKE2 cluster. To summarize the changes: * Update Elasticsearch and Fluentd capabilities in `PodSecurityPolicy` to match `SecurityContext`. * Add missing `PodSecurityPolicy` for all enterprise components. Enterprise components like packetcapture, esgateway, l7-log-collector, etc. are able to be deployed without creating `PodSecurityPolicy` manually. * Manage Prometheus operator `ClusterRole` and `ClusterRoleBinding` in operator. * use `UsePSP` as the only flag to determine whether we need to render `PodSecurityPolicy` or not. [1] tigera#2433
hjiawei · Feb 27, 2023 · 3c54c27 · 3c54c27
1 parent ee9f09f
commit 3c54c27
Show file tree

Hide file tree

Showing 39 changed files with 885 additions and 455 deletions.
diff --git a/pkg/controller/apiserver/apiserver_controller.go b/pkg/controller/apiserver/apiserver_controller.go
@@ -450,6 +450,7 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re
 			ClusterDomain:               r.clusterDomain,
 			ManagementClusterConnection: managementClusterConnection,
 			TrustedBundle:               trustedBundle,
+			UsePSP:                      r.usePSP,
 		}
 		pc := render.PacketCaptureAPI(packetCaptureApiCfg)
 		pcPolicy = render.PacketCaptureAPIPolicy(packetCaptureApiCfg)

diff --git a/pkg/controller/applicationlayer/applicationlayer_controller.go b/pkg/controller/applicationlayer/applicationlayer_controller.go
@@ -91,6 +91,7 @@ func newReconciler(mgr manager.Manager, opts options.AddOptions, licenseAPIReady
 		status:          status.New(mgr.GetClient(), "applicationlayer", opts.KubernetesVersion),
 		clusterDomain:   opts.ClusterDomain,
 		licenseAPIReady: licenseAPIReady,
+		usePSP:          opts.UsePSP,
 	}
 	r.status.Run(opts.ShutdownContext)
 	return r
@@ -162,6 +163,7 @@ type ReconcileApplicationLayer struct {
 	status          status.StatusManager
 	clusterDomain   string
 	licenseAPIReady *utils.ReadyFlag
+	usePSP          bool
 }
 
 // Reconcile reads that state of the cluster for a ApplicationLayer object and makes changes
@@ -280,6 +282,7 @@ func (r *ReconcileApplicationLayer) Reconcile(ctx context.Context, request recon
 		LogRequestsPerInterval: lcSpec.LogRequestsPerInterval,
 		LogIntervalSeconds:     lcSpec.LogIntervalSeconds,
 		ModSecurityConfigMap:   modSecurityRuleSet,
+		UsePSP:                 r.usePSP,
 	}
 	component := applicationlayer.ApplicationLayer(config)
 

diff --git a/pkg/controller/egressgateway/egressgateway_controller.go b/pkg/controller/egressgateway/egressgateway_controller.go
@@ -162,14 +162,12 @@ func (r *ReconcileEgressGateway) Reconcile(ctx context.Context, request reconcil
 	// If there are no Egress Gateway resources, return.
 	ch := utils.NewComponentHandler(log, r.client, r.scheme, nil)
 	if len(egws) == 0 {
-		objects := []client.Object{}
-
+		var objects []client.Object
 		if r.provider == operatorv1.ProviderOpenShift {
-			scc := egressgateway.SecurityContextConstraints()
-			objects = append(objects, scc)
-		} else if r.usePSP {
-			psp := egressgateway.PodSecurityPolicy()
-			objects = append(objects, psp)
+			objects = append(objects, egressgateway.SecurityContextConstraints())
+		}
+		if r.usePSP {
+			objects = append(objects, egressgateway.PodSecurityPolicy())
 		}
 		err := ch.CreateOrUpdateOrDelete(ctx, render.NewDeletionPassthrough(objects...), r.status)
 		if err != nil {

diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go
@@ -1321,7 +1321,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile
 	csiCfg := render.CSIConfiguration{
 		Installation: &instance.Spec,
 		Terminating:  terminating,
-		Openshift:    r.autoDetectedProvider == operator.ProviderOpenShift,
 		UsePSP:       r.usePSP,
 	}
 	components = append(components, render.CSI(&csiCfg))

diff --git a/pkg/controller/logstorage/esgateway.go b/pkg/controller/logstorage/esgateway.go
@@ -1,4 +1,4 @@
-// Copyright (c) 2023 Tigera, Inc. All rights reserved.
+// Copyright (c) 2021-2023 Tigera, Inc. All rights reserved.
 
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -18,7 +18,6 @@ import (
 	"context"
 
 	"github.com/go-logr/logr"
-	"github.com/tigera/operator/pkg/tls/certificatemanagement"
 	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
@@ -28,6 +27,7 @@ import (
 	"github.com/tigera/operator/pkg/controller/utils/imageset"
 	"github.com/tigera/operator/pkg/render"
 	"github.com/tigera/operator/pkg/render/logstorage/esgateway"
+	"github.com/tigera/operator/pkg/tls/certificatemanagement"
 )
 
 func (r *ReconcileLogStorage) createESGateway(
@@ -40,6 +40,7 @@ func (r *ReconcileLogStorage) createESGateway(
 	ctx context.Context,
 	gatewayKeyPair certificatemanagement.KeyPairInterface,
 	trustedBundle certificatemanagement.TrustedBundle,
+	usePSP bool,
 ) (reconcile.Result, bool, error) {
 	// This secret should only ever contain one key.
 	if len(esAdminUserSecret.Data) != 1 {
@@ -67,6 +68,7 @@ func (r *ReconcileLogStorage) createESGateway(
 		ClusterDomain:              r.clusterDomain,
 		EsAdminUserName:            esAdminUserName,
 		ESGatewayKeyPair:           gatewayKeyPair,
+		UsePSP:                     usePSP,
 	}
 
 	esGatewayComponent := esgateway.EsGateway(cfg)

diff --git a/pkg/controller/logstorage/esmetrics.go b/pkg/controller/logstorage/esmetrics.go
@@ -1,4 +1,4 @@
-// Copyright (c) 2023 Tigera, Inc. All rights reserved.
+// Copyright (c) 2021-2023 Tigera, Inc. All rights reserved.
 
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -41,6 +41,7 @@ func (r *ReconcileLogStorage) createESMetrics(
 	hdler utils.ComponentHandler,
 	serverKeyPair certificatemanagement.KeyPairInterface,
 	trustedBundle certificatemanagement.TrustedBundle,
+	usePSP bool,
 ) (reconcile.Result, bool, error) {
 	esMetricsSecret, err := utils.GetSecret(context.Background(), r.client, esmetrics.ElasticsearchMetricsSecret, common.OperatorNamespace())
 	if err != nil {
@@ -60,6 +61,7 @@ func (r *ReconcileLogStorage) createESMetrics(
 		ClusterDomain:        r.clusterDomain,
 		ServerTLS:            serverKeyPair,
 		TrustedBundle:        trustedBundle,
+		UsePSP:               usePSP,
 	}
 	esMetricsComponent := esmetrics.ElasticsearchMetrics(esMetricsCfg)
 	if err = imageset.ApplyImageSet(ctx, r.client, variant, esMetricsComponent); err != nil {

diff --git a/pkg/controller/logstorage/linseed.go b/pkg/controller/logstorage/linseed.go
@@ -41,6 +41,7 @@ func (r *ReconcileLogStorage) createLinseed(
 	ctx context.Context,
 	linseedKeyPair certificatemanagement.KeyPairInterface,
 	trustedBundle certificatemanagement.TrustedBundle,
+	usePSP bool,
 ) (reconcile.Result, bool, error) {
 	// This secret should only ever contain one key.
 	if len(esAdminUserSecret.Data) != 1 {
@@ -61,6 +62,7 @@ func (r *ReconcileLogStorage) createLinseed(
 		ClusterDomain:   r.clusterDomain,
 		KeyPair:         linseedKeyPair,
 		ESAdminUserName: esAdminUserName,
+		UsePSP:          usePSP,
 	}
 
 	linseedComponent := linseed.Linseed(cfg)

diff --git a/pkg/controller/logstorage/logstorage_controller.go b/pkg/controller/logstorage/logstorage_controller.go
@@ -20,25 +20,17 @@ import (
 	"time"
 
 	"github.com/go-logr/logr"
-	"github.com/tigera/operator/pkg/dns"
-	rcertificatemanagement "github.com/tigera/operator/pkg/render/certificatemanagement"
-	"github.com/tigera/operator/pkg/render/logstorage/linseed"
-
-	"github.com/tigera/operator/pkg/render/common/networkpolicy"
-	"github.com/tigera/operator/pkg/render/kubecontrollers"
-	"github.com/tigera/operator/pkg/tls/certificatemanagement"
-	"k8s.io/client-go/kubernetes"
-
-	v3 "github.com/tigera/api/pkg/apis/projectcalico/v3"
 
 	esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1"
 	kbv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/kibana/v1"
-	"github.com/tigera/operator/pkg/controller/certificatemanager"
+	"github.com/elastic/cloud-on-k8s/v2/pkg/utils/stringsutil"
+
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/event"
@@ -49,30 +41,37 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/controller-runtime/pkg/source"
 
-	"github.com/elastic/cloud-on-k8s/v2/pkg/utils/stringsutil"
+	v3 "github.com/tigera/api/pkg/apis/projectcalico/v3"
 	operatorv1 "github.com/tigera/operator/api/v1"
 	"github.com/tigera/operator/pkg/common"
+	"github.com/tigera/operator/pkg/controller/certificatemanager"
 	logstoragecommon "github.com/tigera/operator/pkg/controller/logstorage/common"
 	"github.com/tigera/operator/pkg/controller/options"
 	"github.com/tigera/operator/pkg/controller/status"
 	"github.com/tigera/operator/pkg/controller/utils"
 	"github.com/tigera/operator/pkg/controller/utils/imageset"
+	"github.com/tigera/operator/pkg/dns"
 	"github.com/tigera/operator/pkg/render"
+	rcertificatemanagement "github.com/tigera/operator/pkg/render/certificatemanagement"
 	relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch"
+	"github.com/tigera/operator/pkg/render/common/networkpolicy"
 	rsecret "github.com/tigera/operator/pkg/render/common/secret"
+	"github.com/tigera/operator/pkg/render/kubecontrollers"
 	"github.com/tigera/operator/pkg/render/logstorage/esgateway"
 	"github.com/tigera/operator/pkg/render/logstorage/esmetrics"
+	"github.com/tigera/operator/pkg/render/logstorage/linseed"
 	"github.com/tigera/operator/pkg/render/monitor"
+	"github.com/tigera/operator/pkg/tls/certificatemanagement"
 )
 
-const ResourceName = "log-storage"
-
 var log = logf.Log.WithName("controller_logstorage")
 
 const (
-	defaultEckOperatorMemorySetting  = "512Mi"
 	DefaultElasticsearchStorageClass = "tigera-elasticsearch"
 	LogStorageFinalizer              = "tigera.io/eck-cleanup"
+	ResourceName                     = "log-storage"
+
+	defaultEckOperatorMemorySetting = "512Mi"
 )
 
 // Add creates a new LogStorage Controller and adds it to the Manager. The Manager will set fields on the Controller
@@ -824,6 +823,7 @@ func (r *ReconcileLogStorage) Reconcile(ctx context.Context, request reconcile.R
 			ctx,
 			keyPairs.gateway,
 			trustedBundle,
+			r.usePSP,
 		)
 		if err != nil || !proceed {
 			return result, err
@@ -839,6 +839,7 @@ func (r *ReconcileLogStorage) Reconcile(ctx context.Context, request reconcile.R
 			ctx,
 			keyPairs.linseed,
 			trustedBundle,
+			r.usePSP,
 		)
 		if err != nil || !proceed {
 			return result, err
@@ -864,6 +865,7 @@ func (r *ReconcileLogStorage) Reconcile(ctx context.Context, request reconcile.R
 			hdler,
 			keyPairs.metricsServer,
 			trustedBundle,
+			r.usePSP,
 		)
 		if err != nil || !proceed {
 			return result, err

diff --git a/pkg/controller/monitor/monitor_controller.go b/pkg/controller/monitor/monitor_controller.go
@@ -110,6 +110,7 @@ func newReconciler(mgr manager.Manager, opts options.AddOptions, prometheusReady
 		prometheusReady: prometheusReady,
 		tierWatchReady:  tierWatchReady,
 		clusterDomain:   opts.ClusterDomain,
+		usePSP:          opts.UsePSP,
 	}
 
 	r.status.AddStatefulSets([]types.NamespacedName{
@@ -180,6 +181,7 @@ type ReconcileMonitor struct {
 	prometheusReady *utils.ReadyFlag
 	tierWatchReady  *utils.ReadyFlag
 	clusterDomain   string
+	usePSP          bool
 }
 
 func (r *ReconcileMonitor) getMonitor(ctx context.Context) (*operatorv1.Monitor, error) {
@@ -260,7 +262,7 @@ func (r *ReconcileMonitor) Reconcile(ctx context.Context, request reconcile.Requ
 		r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to create the Tigera CA", err, reqLogger)
 		return reconcile.Result{}, err
 	}
-	serverTLSSecret, err := certificateManager.GetOrCreateKeyPair(r.client, monitor.PrometheusTLSSecretName, common.OperatorNamespace(), dns.GetServiceDNSNames(monitor.PrometheusHTTPAPIServiceName, common.TigeraPrometheusNamespace, r.clusterDomain))
+	serverTLSSecret, err := certificateManager.GetOrCreateKeyPair(r.client, monitor.PrometheusTLSSecretName, common.OperatorNamespace(), dns.GetServiceDNSNames(monitor.PrometheusServiceServiceName, common.TigeraPrometheusNamespace, r.clusterDomain))
 	if err != nil {
 		r.status.SetDegraded(operatorv1.ResourceCreateError, "Error creating TLS certificate", err, reqLogger)
 		return reconcile.Result{}, err
@@ -353,6 +355,7 @@ func (r *ReconcileMonitor) Reconcile(ctx context.Context, request reconcile.Requ
 		TrustedCertBundle:        trustedBundle,
 		Openshift:                r.provider == operatorv1.ProviderOpenShift,
 		KubeControllerPort:       kubeControllersMetricsPort,
+		UsePSP:                   r.usePSP,
 	}
 
 	// Render prometheus component

diff --git a/pkg/controller/utils/discovery.go b/pkg/controller/utils/discovery.go
@@ -195,7 +195,7 @@ func isRKE2(ctx context.Context, c kubernetes.Interface) (bool, error) {
 }
 
 // SupportsPodSecurityPolicies returns true if the cluster contains the policy/v1beta1 PodSecurityPolicy API,
-// and false otherwise. This API is scheuled to be removed in Kubernetes v1.25, but should still be used
+// and false otherwise. This API is scheduled to be removed in Kubernetes v1.25, but should still be used
 // in earlier Kubernetes versions.
 func SupportsPodSecurityPolicies(c kubernetes.Interface) (bool, error) {
 	resources, err := c.Discovery().ServerResourcesForGroupVersion("policy/v1beta1")

diff --git a/pkg/render/apiserver.go b/pkg/render/apiserver.go
@@ -32,7 +32,6 @@ import (
 	operatorv1 "github.com/tigera/operator/api/v1"
 	"github.com/tigera/operator/pkg/components"
 	"github.com/tigera/operator/pkg/controller/k8sapi"
-	"github.com/tigera/operator/pkg/ptr"
 	rcomp "github.com/tigera/operator/pkg/render/common/components"
 	rmeta "github.com/tigera/operator/pkg/render/common/meta"
 	"github.com/tigera/operator/pkg/render/common/networkpolicy"
@@ -186,7 +185,7 @@ func (c *apiServerComponent) Objects() ([]client.Object, []client.Object) {
 	globalObjects, objsToDelete = populateLists(globalObjects, objsToDelete, c.authReaderRoleBinding)
 	globalObjects, objsToDelete = populateLists(globalObjects, objsToDelete, c.webhookReaderClusterRole)
 	globalObjects, objsToDelete = populateLists(globalObjects, objsToDelete, c.webhookReaderClusterRoleBinding)
-	if !c.cfg.Openshift && c.cfg.UsePSP {
+	if c.cfg.UsePSP {
 		globalObjects, objsToDelete = populateLists(globalObjects, objsToDelete, c.apiServerPodSecurityPolicy)
 	}
 
@@ -524,7 +523,7 @@ func (c *apiServerComponent) calicoCustomResourcesClusterRole() *rbacv1.ClusterR
 			},
 		},
 	}
-	if !c.cfg.Openshift {
+	if c.cfg.UsePSP {
 		// Allow access to the pod security policy in case this is enforced on the cluster
 		rules = append(rules, rbacv1.PolicyRule{
 			APIGroups:     []string{"policy"},
@@ -1125,15 +1124,11 @@ func (c *apiServerComponent) apiServerPodSecurityPolicy() (client.Object, client
 		nameToDelete = enterpriseName
 	}
 
-	psp := podsecuritypolicy.NewBasePolicy()
-	psp.GetObjectMeta().SetName(name)
-	psp.Spec.Privileged = false
-	psp.Spec.AllowPrivilegeEscalation = ptr.BoolToPtr(false)
+	psp := podsecuritypolicy.NewBasePolicy(name)
 	psp.Spec.Volumes = append(psp.Spec.Volumes, policyv1beta1.HostPath)
 	psp.Spec.RunAsUser.Rule = policyv1beta1.RunAsUserStrategyRunAsAny
 
-	pspToDelete := podsecuritypolicy.NewBasePolicy()
-	pspToDelete.GetObjectMeta().SetName(nameToDelete)
+	pspToDelete := podsecuritypolicy.NewBasePolicy(nameToDelete)
 
 	return psp, pspToDelete
 }
@@ -1211,7 +1206,7 @@ func (c *apiServerComponent) tigeraCustomResourcesClusterRole() *rbacv1.ClusterR
 			},
 		},
 	}
-	if !c.cfg.Openshift {
+	if c.cfg.UsePSP {
 		// Allow access to the pod security policy in case this is enforced on the cluster
 		rules = append(rules, rbacv1.PolicyRule{
 			APIGroups:     []string{"policy"},