feat: secure metrics endpoints

Signed-off-by: Suleyman Akbas <[email protected]>

Makefile

@@ -42,11 +42,11 @@ ENVTEST_K8S_VERSION = 1.28.0
 
 # TODO: Upgrades > 1.25.3 will lead to createdAt being included in the CSV which currently breaks our change detection. Thus we need to stick to this version until we have a fix.
 OPERATOR_SDK_VERSION ?= 1.25.3
-RBAC_PROXY_IMG ?= gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0
 
 MANAGER_NAME_PREFIX ?= lvms-
 OPERATOR_NAMESPACE ?= openshift-storage
 TOPOLVM_CSI_IMAGE ?= quay.io/lvms_dev/topolvm:latest
+RBAC_PROXY_IMAGE ?= gcr.io/kubebuilder/kube-rbac-proxy:v0.14.1
 CSI_REGISTRAR_IMAGE ?= k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.6.2
 CSI_PROVISIONER_IMAGE ?= k8s.gcr.io/sig-storage/csi-provisioner:v3.3.0
 CSI_LIVENESSPROBE_IMAGE ?= k8s.gcr.io/sig-storage/livenessprobe:v2.8.0
@@ -56,6 +56,7 @@ CSI_SNAPSHOTTER_IMAGE ?= k8s.gcr.io/sig-storage/csi-snapshotter:v6.1.0
 define MANAGER_ENV_VARS
 OPERATOR_NAMESPACE=$(OPERATOR_NAMESPACE)
 TOPOLVM_CSI_IMAGE=$(TOPOLVM_CSI_IMAGE)
+RBAC_PROXY_IMAGE=$(RBAC_PROXY_IMAGE)
 CSI_REGISTRAR_IMAGE=$(CSI_REGISTRAR_IMAGE)
 CSI_PROVISIONER_IMAGE=$(CSI_PROVISIONER_IMAGE)
 CSI_LIVENESSPROBE_IMAGE=$(CSI_LIVENESSPROBE_IMAGE)
@@ -69,13 +70,15 @@ update-mgr-env: ## Feed environment variables to the manager ConfigMap.
 	cp config/default/manager_custom_env.yaml.in config/default/manager_custom_env.yaml
 ifeq ($(UNAME), Darwin)
 	sed -i '' 's|TOPOLVM_CSI_IMAGE_VAL|$(TOPOLVM_CSI_IMAGE)|g' config/default/manager_custom_env.yaml
+	sed -i '' 's|RBAC_PROXY_IMAGE_VAL|$(RBAC_PROXY_IMAGE)|g' config/default/manager_custom_env.yaml
 	sed -i '' 's|CSI_LIVENESSPROBE_IMAGE_VAL|$(CSI_LIVENESSPROBE_IMAGE)|g' config/default/manager_custom_env.yaml
 	sed -i '' 's|CSI_PROVISIONER_IMAGE_VAL|$(CSI_PROVISIONER_IMAGE)|g' config/default/manager_custom_env.yaml
 	sed -i '' 's|CSI_RESIZER_IMAGE_VAL|$(CSI_RESIZER_IMAGE)|g' config/default/manager_custom_env.yaml
 	sed -i '' 's|CSI_REGISTRAR_IMAGE_VAL|$(CSI_REGISTRAR_IMAGE)|g' config/default/manager_custom_env.yaml
 	sed -i '' 's|CSI_SNAPSHOTTER_IMAGE_VAL|$(CSI_SNAPSHOTTER_IMAGE)|g' config/default/manager_custom_env.yaml
 else
 	sed 's|TOPOLVM_CSI_IMAGE_VAL|$(TOPOLVM_CSI_IMAGE)|g' --in-place config/default/manager_custom_env.yaml
+	sed 's|RBAC_PROXY_IMAGE_VAL|$(RBAC_PROXY_IMAGE)|g' --in-place config/default/manager_custom_env.yaml
 	sed 's|CSI_LIVENESSPROBE_IMAGE_VAL|$(CSI_LIVENESSPROBE_IMAGE)|g' --in-place config/default/manager_custom_env.yaml
 	sed 's|CSI_PROVISIONER_IMAGE_VAL|$(CSI_PROVISIONER_IMAGE)|g' --in-place config/default/manager_custom_env.yaml
 	sed 's|CSI_RESIZER_IMAGE_VAL|$(CSI_RESIZER_IMAGE)|g' --in-place config/default/manager_custom_env.yaml
@@ -204,7 +207,6 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified
 
 deploy: update-mgr-env manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} && $(KUSTOMIZE) edit set nameprefix ${MANAGER_NAME_PREFIX}
-	cd config/default && $(KUSTOMIZE) edit set image rbac-proxy=$(RBAC_PROXY_IMG)
 	cd config/webhook && $(KUSTOMIZE) edit set nameprefix ${MANAGER_NAME_PREFIX}
 	$(KUSTOMIZE) build config/default | kubectl apply -f -
 
@@ -236,7 +238,6 @@ bundle: update-mgr-env manifests kustomize operator-sdk rename-csv build-prometh
 	cd config/default && $(KUSTOMIZE) edit set namespace $(OPERATOR_NAMESPACE)
 	cd config/webhook && $(KUSTOMIZE) edit set nameprefix ${MANAGER_NAME_PREFIX}
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} && $(KUSTOMIZE) edit set nameprefix ${MANAGER_NAME_PREFIX}
-	cd config/default && $(KUSTOMIZE) edit set image rbac-proxy=$(RBAC_PROXY_IMG)
 	cd config/manifests/bases && \
 		rm -rf kustomization.yaml && \
 		$(KUSTOMIZE) create --resources $(BUNDLE_PACKAGE).clusterserviceversion.yaml && \

README.md

@@ -350,27 +350,13 @@ $ make undeploy
 
 ## Metrics
 
-The LVM Operator runs a metrics exporter sidecar to export Prometheus metrics. To enable monitoring on OpenShift clusters, assign the `openshift.io/cluster-monitoring` label to the same namespace that you deployed LVMS to.
+To enable monitoring on OpenShift clusters, assign the `openshift.io/cluster-monitoring` label to the same namespace that you deployed LVMS to.
 
 ```bash
 $ oc patch namespace/openshift-storage -p '{"metadata": {"labels": {"openshift.io/cluster-monitoring": "true"}}}'
 ```
 
-Currently, LVMS provides only TopoLVM metrics, which can be accessed either via OpenShift Console or by port-forwarding the relevant service.
-
-```bash
-# port-forward service in one terminal
-$ oc port-forward svc/topolvm-node-metrics 50000:8080
-Forwarding from 127.0.0.1:41685 -> 8080
-Forwarding from [::1]:41685 -> 8080
-...
-...
-
-# in another terminal, view the metrics in localhost using the specified port above
-$ curl -s localhost:50000/metrics | grep -Ei 'topolvm_volumegroup_.*?_bytes\{'
-topolvm_volumegroup_available_bytes{device_class="vg1",node="kube-node"} 4.790222323712e+12
-topolvm_volumegroup_size_bytes{device_class="vg1",node="kube-node"} 4.800959741952e+12
-```
+LVMS provides [TopoLVM metrics](https://github.com/topolvm/topolvm/blob/v0.21.0/docs/topolvm-node.md#prometheus-metrics) and `controller-runtime` metrics, which can be accessed via OpenShift Console.
 
 ## Known Limitations
 

bundle/manifests/lvms-operator-metrics-monitor_monitoring.coreos.com_v1_servicemonitor.yaml

@@ -10,15 +10,24 @@ spec:
   - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
     path: /metrics
     port: topolvm-metrics
-    scheme: http
+    scheme: https
     tlsConfig:
-      insecureSkipVerify: true
+      caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
+      serverName: topolvm-node-metrics.openshift-storage.svc
   - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
     path: /metrics
     port: https
     scheme: https
     tlsConfig:
-      insecureSkipVerify: true
+      caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
+      serverName: lvms-operator-metrics-service.openshift-storage.svc
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    path: /metrics
+    port: vg-manager-https
+    scheme: https
+    tlsConfig:
+      caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
+      serverName: vg-manager-metrics-service.openshift-storage.svc
   selector:
     matchLabels:
       app.kubernetes.io/compose: metrics

bundle/manifests/lvms-operator-metrics-service_v1_service.yaml

@@ -1,6 +1,8 @@
 apiVersion: v1
 kind: Service
 metadata:
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: lvms-operator-metrics-cert
   creationTimestamp: null
   labels:
     app.kubernetes.io/compose: metrics
@@ -9,9 +11,9 @@ metadata:
 spec:
   ports:
   - name: https
-    port: 8443
+    port: 443
     protocol: TCP
-    targetPort: https
+    targetPort: 8443
   selector:
     app.kubernetes.io/name: lvms-operator
 status:

bundle/manifests/lvms-operator.clusterserviceversion.yaml

@@ -557,6 +557,18 @@ spec:
           - get
           - list
           - watch
+        - apiGroups:
+          - authentication.k8s.io
+          resources:
+          - tokenreviews
+          verbs:
+          - create
+        - apiGroups:
+          - authorization.k8s.io
+          resources:
+          - subjectaccessreviews
+          verbs:
+          - create
         serviceAccountName: topolvm-node
       - rules:
         - apiGroups:
@@ -567,6 +579,18 @@ spec:
           - get
           - list
           - watch
+        - apiGroups:
+          - authentication.k8s.io
+          resources:
+          - tokenreviews
+          verbs:
+          - create
+        - apiGroups:
+          - authorization.k8s.io
+          resources:
+          - subjectaccessreviews
+          verbs:
+          - create
         serviceAccountName: vg-manager
       deployments:
       - label:
@@ -590,14 +614,15 @@ spec:
               containers:
               - args:
                 - --health-probe-bind-address=:8081
-                - --metrics-bind-address=127.0.0.1:8080
                 - --leader-elect
                 command:
                 - /lvms
                 - operator
                 env:
                 - name: TOPOLVM_CSI_IMAGE
                   value: quay.io/lvms_dev/topolvm:latest
+                - name: RBAC_PROXY_IMAGE
+                  value: gcr.io/kubebuilder/kube-rbac-proxy:v0.14.1
                 - name: CSI_LIVENESSPROBE_IMAGE
                   value: k8s.gcr.io/sig-storage/livenessprobe:v2.8.0
                 - name: CSI_PROVISIONER_IMAGE
@@ -644,21 +669,9 @@ spec:
                 - mountPath: /tmp/k8s-webhook-server/serving-certs
                   name: cert
                   readOnly: true
-              - args:
-                - --secure-listen-address=0.0.0.0:8443
-                - --upstream=http://127.0.0.1:8080/
-                - --logtostderr=true
-                - --v=0
-                image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0
-                name: kube-rbac-proxy
-                ports:
-                - containerPort: 8443
-                  name: https
-                  protocol: TCP
-                resources:
-                  requests:
-                    cpu: 1m
-                    memory: 20Mi
+                - mountPath: /tmp/k8s-metrics-server/serving-certs
+                  name: metrics-cert
+                  readOnly: true
               securityContext:
                 runAsNonRoot: true
               serviceAccountName: lvms-operator
@@ -668,6 +681,10 @@ spec:
                 secret:
                   defaultMode: 420
                   secretName: lvms-operator-webhook-server-cert
+              - name: metrics-cert
+                secret:
+                  defaultMode: 420
+                  secretName: lvms-operator-metrics-cert
       permissions:
       - rules:
         - apiGroups:

bundle/manifests/topolvm-node-metrics_v1_service.yaml

@@ -1,6 +1,8 @@
 apiVersion: v1
 kind: Service
 metadata:
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: topolvm-metrics-cert
   creationTimestamp: null
   labels:
     app.kubernetes.io/compose: metrics
@@ -9,9 +11,9 @@ metadata:
 spec:
   ports:
   - name: topolvm-metrics
-    port: 8080
+    port: 8443
     protocol: TCP
-    targetPort: 8080
+    targetPort: https
   selector:
     app.kubernetes.io/component: topolvm-node
 status:

bundle/manifests/vg-manager-metrics-service_v1_service.yaml

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: vg-manager-metrics-cert
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/compose: metrics
+    app.kubernetes.io/name: vg-manager
+  name: vg-manager-metrics-service
+spec:
+  ports:
+  - name: vg-manager-https
+    port: 443
+    protocol: TCP
+    targetPort: 8443
+  selector:
+    app.kubernetes.io/name: vg-manager
+status:
+  loadBalancer: {}

cmd/operator/operator.go

@@ -34,6 +34,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 
@@ -49,7 +50,7 @@ import (
 )
 
 const (
-	DefaultMetricsAddr          = ":8080"
+	DefaultDiagnosticsAddr      = ":8443"
 	DefaultProbeAddr            = ":8081"
 	DefaultEnableLeaderElection = false
 )
@@ -58,7 +59,7 @@ type Options struct {
 	Scheme   *runtime.Scheme
 	SetupLog logr.Logger
 
-	metricsAddr          string
+	diagnosticsAddr      string
 	healthProbeAddr      string
 	enableLeaderElection bool
 }
@@ -77,7 +78,7 @@ func NewCmd(opts *Options) *cobra.Command {
 	}
 
 	cmd.Flags().StringVar(
-		&opts.metricsAddr, "metrics-bind-address", DefaultMetricsAddr, "The address the metric endpoint binds to.",
+		&opts.diagnosticsAddr, "diagnostics-address", DefaultDiagnosticsAddr, "The address the diagnostics endpoint binds to.",
 	)
 	cmd.Flags().StringVar(
 		&opts.healthProbeAddr, "health-probe-bind-address", DefaultProbeAddr, "The address the probe endpoint binds to.",
@@ -138,7 +139,9 @@ func run(cmd *cobra.Command, _ []string, opts *Options) error {
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme: opts.Scheme,
 		Metrics: metricsserver.Options{
-			BindAddress: opts.metricsAddr,
+			BindAddress:    opts.diagnosticsAddr,
+			SecureServing:  true,
+			FilterProvider: filters.WithAuthenticationAndAuthorization,
 		},
 		WebhookServer: &webhook.DefaultServer{Options: webhook.Options{
 			Port: 9443,

cmd/vgmanager/vgmanager.go

@@ -38,18 +38,19 @@ import (
 
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 )
 
 const (
-	DefaultMetricsAddr = ":8080"
-	DefaultProbeAddr   = ":8081"
+	DefaultDiagnosticsAddr = ":8443"
+	DefaultProbeAddr       = ":8081"
 )
 
 type Options struct {
 	Scheme   *runtime.Scheme
 	SetupLog logr.Logger
 
-	metricsAddr     string
+	diagnosticsAddr string
 	healthProbeAddr string
 }
 
@@ -67,7 +68,7 @@ func NewCmd(opts *Options) *cobra.Command {
 	}
 
 	cmd.Flags().StringVar(
-		&opts.metricsAddr, "metrics-bind-address", DefaultMetricsAddr, "The address the metric endpoint binds to.",
+		&opts.diagnosticsAddr, "diagnosticsAddr", DefaultDiagnosticsAddr, "The address the diagnostics endpoint binds to.",
 	)
 	cmd.Flags().StringVar(
 		&opts.healthProbeAddr, "health-probe-bind-address", DefaultProbeAddr, "The address the probe endpoint binds to.",
@@ -80,7 +81,9 @@ func run(_ *cobra.Command, _ []string, opts *Options) error {
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme: opts.Scheme,
 		Metrics: metricsserver.Options{
-			BindAddress: opts.metricsAddr,
+			BindAddress:    opts.diagnosticsAddr,
+			SecureServing:  true,
+			FilterProvider: filters.WithAuthenticationAndAuthorization,
 		},
 		WebhookServer: &webhook.DefaultServer{Options: webhook.Options{
 			Port: 9443,

config/default/kustomization.yaml

@@ -16,8 +16,8 @@ namespace: openshift-storage
 # If you want your controller-manager to expose the /metrics
 # endpoint w/o any authn/z, please comment the following line.
 patchesStrategicMerge:
-- manager_auth_proxy_patch.yaml
 - manager_custom_env.yaml
+- manager_metrics_patch.yaml
 - manager_webhook_patch.yaml
 
 # Mount the controller config file for loading manager configurations
@@ -33,7 +33,3 @@ resources:
 - ../manager
 - ../prometheus
 - ../webhook
-images:
-- name: rbac-proxy
-  newName: gcr.io/kubebuilder/kube-rbac-proxy
-  newTag: v0.8.0