Support sriov-network-metrics-exporter

Deploy `sriov-network-metrics-exporter` DaemonSet and related configuration. The feature is activated by the feature gate `metricsExporter`. Add deployment logic to the SriovOperatorConfig reconcile loop. The operator's environment variable `SRIOV_NETWORK_METRICS_EXPORTER_IMAGE` controls the exporter image to deploy. Update helm charts with `.Values.images.metricsExporter` with the same semantic. Signed-off-by: Andrea Panattoni <[email protected]>
k8snetworkplumbingwg · Apr 2, 2024 · ab693dc · ab693dc
1 parent 160d177
commit ab693dc
Show file tree

Hide file tree

Showing 15 changed files with 424 additions and 13 deletions.
diff --git a/bindata/manifests/metrics-exporter/metrics-config-map.yaml b/bindata/manifests/metrics-exporter/metrics-config-map.yaml
@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: sriov-network-metrics-exporter-config
+  namespace: {{.Namespace}}
+data:
+  drivers.yaml: |-
+    drivers:
+      - name: ice
+        version: 1.9.11
+      - name: mlx5_core
+        version: 5.15.0-53-generic
diff --git a/bindata/manifests/metrics-exporter/metrics-daemonset.yaml b/bindata/manifests/metrics-exporter/metrics-daemonset.yaml
@@ -0,0 +1,122 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    app: sriov-network-metrics-exporter
+  name: sriov-network-metrics-exporter
+  namespace: {{.Namespace}}
+spec:
+  selector:
+    matchLabels:
+      app: sriov-network-metrics-exporter
+  template:
+    metadata:
+      labels:
+        app: sriov-network-metrics-exporter
+    spec:
+      hostNetwork: true
+      serviceAccountName: sriov-network-config-daemon
+      {{- if .ImagePullSecrets }}
+      imagePullSecrets:
+      {{- range .ImagePullSecrets }}
+      - name: {{ . }}
+      {{- end }}
+      {{- end }}
+      containers:
+      - args:
+        - --web.listen-address=127.0.0.1:{{.MetricsExporterPort}}
+        - --path.kubecgroup=/sys/fs/cgroup
+        - --path.sysbuspci=/host/sys/bus/pci/devices/
+        - --path.sysclassnet=/host/sys/class/net/
+        - --path.cpucheckpoint=/host/cpu_manager_state
+        - --path.kubeletsocket=/host/kubelet.sock
+        - --collector.kubepoddevice=true
+        - --collector.vfstatspriority=sysfs,netlink
+        image: {{.Image}}
+        imagePullPolicy: IfNotPresent
+        name: metrics-exporter
+        resources:
+          requests:
+            memory: 100Mi
+            cpu: 100m
+        securityContext:
+          capabilities:
+            drop:
+              - ALL      
+          readOnlyRootFilesystem: true
+          allowPrivilegeEscalation: false
+        volumeMounts:
+        - mountPath: /host/kubelet.sock
+          name: kubeletsocket
+        - mountPath: /host/sys/bus/pci/devices
+          name: sysbuspcidevices
+          readOnly: true
+        - mountPath: /host/sys/devices
+          name: sysdevices
+          readOnly: true
+        - mountPath: /host/sys/class/net
+          name: sysclassnet
+          readOnly: true
+        - mountPath: /host/cpu_manager_state
+          name: cpucheckpoint
+          readOnly: true
+        - name: sriov-network-metrics-exporter-config
+          mountPath: /etc/sriov-network-metrics-exporter
+      - name: kube-rbac-proxy
+        image: '{{.KubeRbacProxyImage}}'
+        imagePullPolicy: IfNotPresent
+        args:
+          - --logtostderr
+          - --secure-listen-address=[$(HOST_IP)]:{{.MetricsExporterPort}}
+          - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256
+          - --upstream=http://127.0.0.1:{{.MetricsExporterPort}}/
+          - --tls-private-key-file=/etc/metrics/tls.key
+          - --tls-cert-file=/etc/metrics/tls.crt
+        ports:
+          - containerPort: {{.MetricsExporterPort}}
+            name: https-metrics
+        env:
+          - name: HOST_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.hostIP
+        resources:
+          requests:
+            cpu: 10m
+            memory: 20Mi
+        volumeMounts:
+          - name: metrics-certs
+            mountPath: /etc/metrics
+            readOnly: true
+      nodeSelector:
+        kubernetes.io/os: linux
+        node-role.kubernetes.io/worker: ""
+      restartPolicy: Always
+      volumes:
+      - hostPath:
+          path: /var/lib/kubelet/pod-resources/kubelet.sock
+          type: "Socket"
+        name: kubeletsocket
+      - hostPath:
+          path: /var/lib/kubelet/cpu_manager_state
+          type: "File"
+        name: cpucheckpoint
+      - hostPath:
+          path: /sys/class/net
+          type: "Directory"
+        name: sysclassnet
+      - hostPath:
+          path: /sys/bus/pci/devices
+          type: "Directory"
+        name: sysbuspcidevices
+      - hostPath:
+          path: /sys/devices
+          type: "Directory"
+        name: sysdevices
+      - name: sriov-network-metrics-exporter-config
+        configMap:
+          name: sriov-network-metrics-exporter-config
+      - name: metrics-certs
+        secret:
+          defaultMode: 420
+          secretName: {{ .MetricsExporterSecretName }}
diff --git a/bindata/manifests/metrics-exporter/metrics-rbac.yaml b/bindata/manifests/metrics-exporter/metrics-rbac.yaml
@@ -0,0 +1,66 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: metrics-exporter-sa
+  namespace: {{.Namespace}}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: metrics-exporter-role
+  namespace: {{.Namespace}}
+rules:
+  - apiGroups:
+      - security.openshift.io
+    resourceNames:
+      - hostaccess
+    resources:
+      - securitycontextconstraints
+    verbs:
+      - use
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: metrics-exporter-rb
+  namespace: {{.Namespace}}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: metrics-exporter-role
+subjects:
+  - kind: ServiceAccount
+    name: metrics-exporter-sa
+    namespace: {{.Namespace}}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: sriov-metrics-kube-rbac-role
+rules:
+- apiGroups:
+  - authentication.k8s.io
+  resources:
+  - tokenreviews
+  verbs:
+  - create
+- apiGroups:
+  - authorization.k8s.io
+  resources:
+  - subjectaccessreviews
+  verbs:
+  - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: sriov-metrics-kube-rbac-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: sriov-metrics-kube-rbac-role
+subjects:
+- kind: ServiceAccount
+  name: metrics-exporter-sa
+  namespace: {{.Namespace}}
diff --git a/bindata/manifests/metrics-exporter/metrics-service.yaml b/bindata/manifests/metrics-exporter/metrics-service.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: sriov-network-metrics-exporter-service
+  namespace: {{.Namespace}}
+  annotations:
+    prometheus.io/target: "true"
+    service.beta.openshift.io/serving-cert-secret-name: {{ .MetricsExporterSecretName }}
+spec:
+  selector:
+    app.kubernetes.io/name: sriov-metrics-exporter
+  ports:
+    - protocol: TCP
+      port: {{ .MetricsExporterPort }}
+      targetPort: {{ .MetricsExporterPort }}
diff --git a/controllers/sriovoperatorconfig_controller.go b/controllers/sriovoperatorconfig_controller.go
@@ -117,6 +117,10 @@ func (r *SriovOperatorConfigReconciler) Reconcile(ctx context.Context, req ctrl.
 		return reconcile.Result{}, err
 	}
 
+	if err = r.syncMetricsExporter(ctx, defaultConfig); err != nil {
+		return reconcile.Result{}, err
+	}
+
 	// For Openshift we need to create the systemd files using a machine config
 	if vars.ClusterType == consts.ClusterTypeOpenshift {
 		// TODO: add support for hypershift as today there is no MCO on hypershift clusters
@@ -162,7 +166,6 @@ func (r *SriovOperatorConfigReconciler) syncConfigDaemonSet(ctx context.Context,
 	data.Data["Namespace"] = vars.Namespace
 	data.Data["SRIOVCNIImage"] = os.Getenv("SRIOV_CNI_IMAGE")
 	data.Data["SRIOVInfiniBandCNIImage"] = os.Getenv("SRIOV_INFINIBAND_CNI_IMAGE")
-	data.Data["ReleaseVersion"] = os.Getenv("RELEASEVERSION")
 	data.Data["ClusterType"] = vars.ClusterType
 	data.Data["DevMode"] = os.Getenv("DEV_MODE")
 	data.Data["ImagePullSecrets"] = GetImagePullSecrets()
@@ -196,27 +199,89 @@ func (r *SriovOperatorConfigReconciler) syncConfigDaemonSet(ctx context.Context,
 	}
 	// Sync DaemonSets
 	for _, obj := range objs {
-		if obj.GetKind() == "DaemonSet" && len(dc.Spec.ConfigDaemonNodeSelector) > 0 {
-			scheme := kscheme.Scheme
-			ds := &appsv1.DaemonSet{}
-			err = scheme.Convert(obj, ds, nil)
+		err = updateDaemonsetNodeSelector(obj, dc.Spec.ConfigDaemonNodeSelector)
+		if err != nil {
+			return err
+		}
+
+		err = r.syncK8sResource(ctx, dc, obj)
+		if err != nil {
+			logger.Error(err, "Couldn't sync SR-IoV daemons objects")
+			return err
+		}
+	}
+	return nil
+}
+
+func updateDaemonsetNodeSelector(obj *uns.Unstructured, nodeSelector map[string]string) error {
+	if obj.GetKind() != "DaemonSet" {
+		return nil
+	}
+
+	if len(nodeSelector) == 0 {
+		return nil
+	}
+
+	ds := &appsv1.DaemonSet{}
+	scheme := kscheme.Scheme
+	err := scheme.Convert(obj, ds, nil)
+	if err != nil {
+		return fmt.Errorf("failed to convert Unstructured [%s] to DaemonSet: %v", obj.GetName(), err)
+	}
+
+	ds.Spec.Template.Spec.NodeSelector = nodeSelector
+
+	err = scheme.Convert(ds, obj, nil)
+	if err != nil {
+		return fmt.Errorf("failed to convert DaemonSet [%s] to Unstructured: %v", obj.GetName(), err)
+	}
+	return nil
+}
+
+func (r *SriovOperatorConfigReconciler) syncMetricsExporter(ctx context.Context, dc *sriovnetworkv1.SriovOperatorConfig) error {
+	logger := log.Log.WithName("syncMetricsExporter")
+	logger.V(1).Info("Start to sync metrics exporter")
+
+	data := render.MakeRenderData()
+	data.Data["Image"] = os.Getenv("SRIOV_NETWORK_METRICS_EXPORTER_IMAGE")
+	data.Data["Namespace"] = vars.Namespace
+	data.Data["ReleaseVersion"] = os.Getenv("RELEASEVERSION")
+	data.Data["ImagePullSecrets"] = GetImagePullSecrets()
+	data.Data["MetricsExporterSecretName"] = os.Getenv("METRICS_EXPORTER_SECRET_NAME")
+	data.Data["MetricsExporterPort"] = os.Getenv("METRICS_EXPORTER_PORT")
+	data.Data["KubeRbacProxyImage"] = os.Getenv("KUBE_RBAC_PROXY_IMAGE")
+
+	objs, err := render.RenderDir(consts.MetricsExporterPath, &data)
+	if err != nil {
+		logger.Error(err, "Fail to render metrics exporter manifests")
+		return err
+	}
+
+	deployMetricsExporter, ok := dc.Spec.FeatureGates[consts.MetricsExporterFeatureGate]
+	if ok && deployMetricsExporter {
+		for _, obj := range objs {
+
+			err = updateDaemonsetNodeSelector(obj, dc.Spec.ConfigDaemonNodeSelector)
 			if err != nil {
-				logger.Error(err, "Fail to convert to DaemonSet")
 				return err
 			}
-			ds.Spec.Template.Spec.NodeSelector = dc.Spec.ConfigDaemonNodeSelector
-			err = scheme.Convert(ds, obj, nil)
+
+			err = r.syncK8sResource(ctx, dc, obj)
 			if err != nil {
-				logger.Error(err, "Fail to convert to Unstructured")
+				logger.Error(err, "Couldn't sync metrics exporter objects")
 				return err
 			}
 		}
-		err = r.syncK8sResource(ctx, dc, obj)
+		return nil
+	}
+
+	for _, obj := range objs {
+		err = r.deleteK8sResource(ctx, obj)
 		if err != nil {
-			logger.Error(err, "Couldn't sync SR-IoV daemons objects")
 			return err
 		}
 	}
+
 	return nil
 }
 

diff --git a/controllers/sriovoperatorconfig_controller_test.go b/controllers/sriovoperatorconfig_controller_test.go
@@ -7,6 +7,7 @@ import (
 
 	admv1 "k8s.io/api/admissionregistration/v1"
 	appsv1 "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -325,5 +326,34 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() {
 			})
 			Expect(err).ToNot(HaveOccurred())
 		})
+		It("should deploy the metrics-exporter when the feature gate is enabled", func() {
+			config := &sriovnetworkv1.SriovOperatorConfig{}
+			Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: testNamespace, Name: "default"}, config)).NotTo(HaveOccurred())
+
+			daemonSet := &appsv1.DaemonSet{}
+			err := k8sClient.Get(ctx, types.NamespacedName{Name: "sriov-metrics-exporter", Namespace: testNamespace}, daemonSet)
+			Expect(err).To(HaveOccurred())
+			Expect(errors.IsNotFound(err)).To(BeTrue())
+
+			config.Spec.FeatureGates = map[string]bool{constants.MetricsExporterFeatureGate: true}
+			err = k8sClient.Update(ctx, config)
+			Expect(err).NotTo(HaveOccurred())
+
+			DeferCleanup(func() {
+				config.Spec.FeatureGates = map[string]bool{}
+				err = k8sClient.Update(ctx, config)
+				Expect(err).NotTo(HaveOccurred())
+			})
+
+			err = util.WaitForNamespacedObject(&appsv1.DaemonSet{}, k8sClient, testNamespace, "sriov-network-metrics-exporter", util.RetryInterval, util.APITimeout)
+			Expect(err).NotTo(HaveOccurred())
+
+			err = util.WaitForNamespacedObject(&v1.ConfigMap{}, k8sClient, testNamespace, "sriov-network-metrics-exporter-config", util.RetryInterval, util.APITimeout)
+			Expect(err).NotTo(HaveOccurred())
+
+			err = util.WaitForNamespacedObject(&v1.Service{}, k8sClient, testNamespace, "sriov-network-metrics-exporter-service", util.RetryInterval, util.APITimeout)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
 	})
 })
diff --git a/controllers/suite_test.go b/controllers/suite_test.go
@@ -121,6 +121,14 @@ var _ = BeforeSuite(func() {
 	Expect(err).NotTo(HaveOccurred())
 	err = os.Setenv("OPERATOR_NAME", "sriov-network-operator")
 	Expect(err).NotTo(HaveOccurred())
+	err = os.Setenv("SRIOV_NETWORK_METRICS_EXPORTER_IMAGE", "mock-image")
+	Expect(err).NotTo(HaveOccurred())
+	err = os.Setenv("METRICS_EXPORTER_SECRET_NAME", "metrics-exporter-cert")
+	Expect(err).NotTo(HaveOccurred())
+	err = os.Setenv("METRICS_EXPORTER_PORT", "9110")
+	Expect(err).NotTo(HaveOccurred())
+	err = os.Setenv("KUBE_RBAC_PROXY_IMAGE", "mock-image")
+	Expect(err).NotTo(HaveOccurred())
 
 	By("bootstrapping test environment")
 	testEnv = &envtest.Environment{