feat: add autoscaler admission webhook

Dockerfile

@@ -12,6 +12,7 @@ RUN if [ "$USERNAME" = "ng-user" ]; then \
 ADD bin/${TARGETDIR}/controller-manager /usr/local/bin/controller-manager
 ADD bin/${TARGETDIR}/autoscaler /usr/local/bin/autoscaler
 ADD bin/${TARGETDIR}/scheduler /usr/local/bin/scheduler
+ADD bin/${TARGETDIR}/certificate-generator /usr/local/bin/certificate-generator
 
 # [Optional] Set the default user. Omit if you want to keep the default as root.
 USER $USERNAME

Makefile

@@ -79,6 +79,7 @@ build-operator: ## Build operator related binary.
 	$(GO_BUILD) -ldflags '$(LDFLAGS)' -o bin/$(TARGETDIR)/controller-manager cmd/controller-manager/main.go
 	$(GO_BUILD) -ldflags '$(LDFLAGS)' -o bin/$(TARGETDIR)/autoscaler cmd/autoscaler/main.go
 	$(GO_BUILD) -ldflags '$(LDFLAGS)' -o bin/$(TARGETDIR)/scheduler cmd/scheduler/main.go
+	$(GO_BUILD) -ldflags '$(LDFLAGS)' -o bin/$(TARGETDIR)/certificate-generator cmd/certificate-generator/main.go
 
 build-provisioner: ## Build provisioner binary.
 	$(GO_BUILD) -ldflags '$(LDFLAGS)' -o bin/$(TARGETDIR)/local-pv-provisioner cmd/provisioner/main.go

charts/nebula-operator/templates/_helpers.tpl

@@ -97,6 +97,20 @@ Admission webhook name of the chart.
 {{ include "nebula-operator.name" . }}-webhook
 {{- end }}
 
+{{/*
+Controller Manager Admission webhook name.
+*/}}
+{{- define "controller-manager-admission-webhook.name" -}}
+controller-manager-{{ include "nebula-operator.name" . }}-webhook
+{{- end }}
+
+{{/*
+Autoscaler Admission webhook name.
+*/}}
+{{- define "autoscaler-admission-webhook.name" -}}
+autoscaler-{{ include "nebula-operator.name" . }}-webhook
+{{- end }}
+
 {{/*
 Admission webhook selector labels
 */}}

charts/nebula-operator/templates/admission-webhook-certificate.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.admissionWebhook.create }}
+{{- if and (or .Values.admissionWebhook.contollerManagerAdmissionWebhook.create .Values.admissionWebhook.autoscalerAdmissionWebhook.create) .Values.admissionWebhook.useCertManager }}
 ---
 apiVersion: cert-manager.io/v1
 kind: Certificate
@@ -9,8 +9,10 @@ metadata:
   {{- include "admission-webhook.labels" . | nindent 4 }}
 spec:
   dnsNames:
-    - {{ template "admission-webhook.name" . }}-service.{{ template "nebula-operator.namespace" . }}.svc
-    - {{ template "admission-webhook.name" . }}-service.{{ template "nebula-operator.namespace" . }}.svc.{{ default "cluster.local" .Values.kubernetesClusterDomain }}
+    - {{ template "controller-manager-admission-webhook.name" . }}-service.{{ template "nebula-operator.namespace" . }}.svc
+    - {{ template "controller-manager-admission-webhook.name" . }}-service.{{ template "nebula-operator.namespace" . }}.svc.{{ default "cluster.local" .Values.kubernetesClusterDomain }}
+    - {{ template "autoscaler-admission-webhook.name" . }}-service.{{ template "nebula-operator.namespace" . }}.svc
+    - {{ template "autoscaler-admission-webhook.name" . }}-service.{{ template "nebula-operator.namespace" . }}.svc.{{ default "cluster.local" .Values.kubernetesClusterDomain }}
   issuerRef:
     kind: Issuer
     name: {{ template "admission-webhook.name" . }}-issuer

charts/nebula-operator/templates/admission-webhook-registration.yaml

@@ -1,20 +1,23 @@
-{{- if .Values.admissionWebhook.create }}
+{{- if or .Values.admissionWebhook.contollerManagerAdmissionWebhook.create .Values.admissionWebhook.autoscalerAdmissionWebhook.create }}
 ---
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration
 metadata:
+  {{- if .Values.admissionWebhook.useCertManager }}
   annotations:
     cert-manager.io/inject-ca-from: {{ template "nebula-operator.namespace" . }}/{{ template "admission-webhook.name" . }}-cert
+  {{- end }}
   name: {{ template "admission-webhook.name" . }}-validating
   labels:
   {{- include "admission-webhook.labels" . | nindent 4 }}
 webhooks:
+  {{- if .Values.admissionWebhook.contollerManagerAdmissionWebhook.create }}
   - name: nebulaclustervalidating.nebula-graph.io
     admissionReviewVersions:
       - v1
     clientConfig:
       service:
-        name: {{ template "admission-webhook.name" . }}-service
+        name: {{ template "controller-manager-admission-webhook.name" . }}-service
         namespace: {{ template "nebula-operator.namespace" . }}
         path: /validate-nebulacluster
     failurePolicy: Fail
@@ -32,5 +35,31 @@ webhooks:
         scope: "*"
     sideEffects: None
     timeoutSeconds: 3
+  {{- end }}
+
+  {{- if .Values.admissionWebhook.autoscalerAdmissionWebhook.create }}
+  - name: nebulaautoscalingvalidating.nebula-graph.io
+    admissionReviewVersions:
+      - v1
+    clientConfig:
+      service:
+        name: {{ template "autoscaler-admission-webhook.name" . }}-service
+        namespace: {{ template "nebula-operator.namespace" . }}
+        path: /validate-nebulaautoscaler
+    failurePolicy: Fail
+    rules:
+      - apiGroups:
+          - autoscaling.nebula-graph.io
+        apiVersions:
+          - v1alpha1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - nebulaautoscalers
+        scope: "*"
+    sideEffects: None
+    timeoutSeconds: 3
+  {{- end }}
 {{- end }}
 

charts/nebula-operator/templates/autoscaler-admission-webhook-service.yaml

@@ -0,0 +1,17 @@
+{{- if .Values.admissionWebhook.autoscalerAdmissionWebhook.create }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "autoscaler-admission-webhook.name" . }}-service
+  namespace: {{ template "nebula-operator.namespace" . }}
+  labels:
+  {{- include "admission-webhook.labels" . | nindent 4 }}
+spec:
+  ports:
+    - port: 443
+      targetPort: 9448
+  selector:
+  {{- include "admission-webhook.matchLabels" . | nindent 4 }}
+{{- end }}
+

charts/nebula-operator/templates/admission-webhook-service.yaml → charts/nebula-operator/templates/controller-manager-admission-webhook-service.yaml

@@ -1,9 +1,9 @@
-{{- if .Values.admissionWebhook.create }}
+{{- if .Values.admissionWebhook.contollerManagerAdmissionWebhook.create }}
 ---
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ template "admission-webhook.name" . }}-service
+  name: {{ template "controller-manager-admission-webhook.name" . }}-service
   namespace: {{ template "nebula-operator.namespace" . }}
   labels:
   {{- include "admission-webhook.labels" . | nindent 4 }}

charts/nebula-operator/templates/controller-manager-deployment.yaml

@@ -1,4 +1,17 @@
 {{- if .Values.controllerManager.create }}
+{{- if and (or .Values.admissionWebhook.contollerManagerAdmissionWebhook.create .Values.admissionWebhook.autoscalerAdmissionWebhook.create) (not .Values.admissionWebhook.useCertManager) }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ template "admission-webhook.name" . }}-secret
+  namespace: {{ template "nebula-operator.namespace" . }}
+type: kubernetes.io/tls
+data:
+  tls.crt: ""
+  tls.key: ""
+  ca.crt: ""
+{{- end }}
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -19,10 +32,42 @@ spec:
     spec:
       serviceAccountName: {{ template "controller-manager.name" . }}-sa
       {{- include "nebula-operator.imagePullSecrets" . | indent 6 }}
+      {{- if and (or .Values.admissionWebhook.contollerManagerAdmissionWebhook.create .Values.admissionWebhook.autoscalerAdmissionWebhook.create) (not .Values.admissionWebhook.useCertManager) }}
+      initContainers:
+        - name: cert-init
+          image: {{ .Values.image.nebulaOperator.image }}
+          imagePullPolicy: {{ .Values.image.nebulaOperator.imagePullPolicy }}
+          command:
+            - /usr/local/bin/certificate-generator
+          args:
+            - --v={{ .Values.controllerManager.verbosity }}
+            - --webhook-namespace={{ template "nebula-operator.namespace" . }}
+            - --webhook-server-name={{ template "admission-webhook.name" . }}-validating
+            - --webhook-names={{ template "controller-manager-admission-webhook.name" . }}-service,{{ template "autoscaler-admission-webhook.name" . }}-service
+            - --certificate-dir=/tmp/k8s-webhook-server/serving-certs
+            - --certificate-validity={{ .Values.admissionWebhook.certValidity }}
+            - --secret-namespace={{ template "nebula-operator.namespace" . }}
+            - --secret-name={{ template "admission-webhook.name" . }}-secret
+            - --kube-domain={{ default "cluster.local" .Values.kubernetesClusterDomain }}
+            - --init-only=true
+          env:
+            {{- if .Values.kubernetesClusterDomain }}
+            - name: KUBERNETES_CLUSTER_DOMAIN
+              value: {{ .Values.kubernetesClusterDomain }}
+            {{- end }}
+          volumeMounts:
+            - mountPath: /tmp/k8s-webhook-server/serving-certs
+              name: cert
+              readOnly: true
+      {{- with .Values.controllerManager.extraInitContainers }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- else}}
       {{- with .Values.controllerManager.extraInitContainers }}
       initContainers:
       {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- end}}
       containers:
         - name: controller-manager
           image: {{ .Values.image.nebulaOperator.image }}
@@ -43,8 +88,8 @@ spec:
             - --concurrent-nebulabackup-syncs={{ .Values.concurrentNebulaBackupSyncs }}
             - --leader-elect
             - --leader-elect-resource-namespace={{ template "nebula-operator.namespace" . }}
-            - --enable-admission-webhook={{ .Values.admissionWebhook.create }}
-            - --webhook-secure-port={{ .Values.admissionWebhook.webhookBindPort }}
+            - --enable-admission-webhook={{ .Values.admissionWebhook.contollerManagerAdmissionWebhook.create }}
+            - --webhook-secure-port={{ .Values.admissionWebhook.contollerManagerAdmissionWebhook.webhookBindPort }}
             - --enable-kruise-scheme={{ .Values.enableKruiseScheme }}
             - --v={{ .Values.controllerManager.verbosity }}
           {{- if or .Values.kubernetesClusterDomain .Values.controllerManager.env }}
@@ -55,9 +100,9 @@ spec:
           {{- end }}
           {{- if .Values.controllerManager.env }}{{ toYaml .Values.controllerManager.env | nindent 12 }}{{- end }}
           {{- end }}
-          {{- if .Values.admissionWebhook.create }}
+          {{- if .Values.admissionWebhook.contollerManagerAdmissionWebhook.create }}
           ports:
-            - containerPort: {{ .Values.admissionWebhook.webhookBindPort | default 9443 }}
+            - containerPort: {{ .Values.admissionWebhook.contollerManagerAdmissionWebhook.webhookBindPort | default 9443 }}
               name: webhook-server
               protocol: TCP
           {{- end }}
@@ -77,9 +122,9 @@ spec:
             periodSeconds: 10
           securityContext:
             allowPrivilegeEscalation: false
-          {{- if or .Values.controllerManager.extraVolumeMounts .Values.admissionWebhook.create }}
+          {{- if or .Values.controllerManager.extraVolumeMounts .Values.admissionWebhook.contollerManagerAdmissionWebhook.create }}
           volumeMounts:
-          {{- if .Values.admissionWebhook.create }}
+          {{- if .Values.admissionWebhook.contollerManagerAdmissionWebhook.create }}
             - mountPath: /tmp/k8s-webhook-server/serving-certs
               name: cert
               readOnly: true
@@ -100,6 +145,8 @@ spec:
             - --leader-elect
             - --leader-elect-resource-namespace={{ template "nebula-operator.namespace" . }}
             - --v={{ .Values.controllerManager.verbosity }}
+            - --enable-admission-webhook={{ .Values.admissionWebhook.autoscalerAdmissionWebhook.create }}
+            - --webhook-secure-port={{ .Values.admissionWebhook.autoscalerAdmissionWebhook.webhookBindPort }}
           {{- if or .Values.kubernetesClusterDomain .Values.controllerManager.env }}
           env:
           {{- if .Values.kubernetesClusterDomain }}
@@ -108,6 +155,12 @@ spec:
           {{- end }}
           {{- if .Values.controllerManager.env }}{{ toYaml .Values.controllerManager.env | nindent 12 }}{{- end }}
           {{- end }}
+          {{- if .Values.admissionWebhook.autoscalerAdmissionWebhook.create }}
+          ports:
+            - containerPort: {{ .Values.admissionWebhook.autoscalerAdmissionWebhook.webhookBindPort | default 9448 }}
+              name: webhook-server
+              protocol: TCP
+          {{- end }}
           resources:
           {{- toYaml .Values.controllerManager.resources | nindent 12 }}
           livenessProbe:
@@ -124,6 +177,40 @@ spec:
             periodSeconds: 10
           securityContext:
             allowPrivilegeEscalation: false
+          {{- if .Values.admissionWebhook.autoscalerAdmissionWebhook.create }}
+          volumeMounts:
+            - mountPath: /tmp/k8s-webhook-server/serving-certs
+              name: cert
+              readOnly: true
+          {{- end }}
+        {{- if and (or .Values.admissionWebhook.contollerManagerAdmissionWebhook.create .Values.admissionWebhook.autoscalerAdmissionWebhook.create) (not .Values.admissionWebhook.useCertManager) }}
+        - name: cert-rotation
+          image: {{ .Values.image.nebulaOperator.image }}
+          imagePullPolicy: {{ .Values.image.nebulaOperator.imagePullPolicy }}
+          command:
+            - /usr/local/bin/certificate-generator
+          args:
+            - --leader-elect
+            - --leader-elect-resource-namespace={{ template "nebula-operator.namespace" . }}
+            - --v={{ .Values.controllerManager.verbosity }}
+            - --webhook-namespace={{ template "nebula-operator.namespace" . }}
+            - --webhook-server-name={{ template "admission-webhook.name" . }}-validating
+            - --webhook-names={{ template "controller-manager-admission-webhook.name" . }}-service,{{ template "autoscaler-admission-webhook.name" . }}-service
+            - --certificate-dir=/tmp/k8s-webhook-server/serving-certs
+            - --certificate-validity={{ .Values.admissionWebhook.certValidity }}
+            - --secret-namespace={{ template "nebula-operator.namespace" . }}
+            - --secret-name={{ template "admission-webhook.name" . }}-secret
+            - --kube-domain={{ default "cluster.local" .Values.kubernetesClusterDomain }}
+          env:
+            {{- if .Values.kubernetesClusterDomain }}
+            - name: KUBERNETES_CLUSTER_DOMAIN
+              value: {{ .Values.kubernetesClusterDomain }}
+            {{- end }}
+          volumeMounts:
+            - mountPath: /tmp/k8s-webhook-server/serving-certs
+              name: cert
+              readOnly: true
+        {{- end}}
     {{- with .Values.controllerManager.sidecarContainers }}
       {{- range $name, $spec :=  $.Values.controllerManager.sidecarContainers }}
         - name: {{ $name }}
@@ -153,9 +240,9 @@ spec:
       tolerations:
       {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- if or .Values.controllerManager.extraVolumes .Values.admissionWebhook.create }}
+      {{- if or .Values.controllerManager.extraVolumes (or .Values.admissionWebhook.contollerManagerAdmissionWebhook.create .Values.admissionWebhook.autoscalerAdmissionWebhook.create) }}
       volumes:
-      {{- if .Values.admissionWebhook.create }}
+      {{- if or .Values.admissionWebhook.contollerManagerAdmissionWebhook.create .Values.admissionWebhook.autoscalerAdmissionWebhook.create}}
       - name: cert
         secret:
           defaultMode: 420

charts/nebula-operator/templates/controller-manager-rbac.yaml

@@ -37,6 +37,19 @@ rules:
     verbs:
       - create
       - patch
+  {{- if and (or .Values.admissionWebhook.contollerManagerAdmissionWebhook.create .Values.admissionWebhook.autoscalerAdmissionWebhook.create) (not .Values.admissionWebhook.useCertManager) }}
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - patch
+  {{- end}}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
@@ -387,6 +400,18 @@ rules:
     verbs:
       - get
       - list
+  {{- if and (or .Values.admissionWebhook.contollerManagerAdmissionWebhook.create .Values.admissionWebhook.autoscalerAdmissionWebhook.create) (not .Values.admissionWebhook.useCertManager) }}
+  - apiGroups:
+      - admissionregistration.k8s.io
+    resources:
+      - validatingwebhookconfigurations
+    verbs:
+      - get
+      - list
+      - watch
+      - update
+      - patch
+  {{- end }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

charts/nebula-operator/values.yaml

@@ -64,9 +64,17 @@ controllerManager:
 #    runAsNonRoot: true
 
 admissionWebhook:
-  create: false
-  # The TCP port the Webhook server binds to. (default 9443)
-  webhookBindPort: 9443
+  contollerManagerAdmissionWebhook:
+    create: false
+    # The TCP port the Webhook server binds to. (default 9443)
+    webhookBindPort: 9443
+  autoscalerAdmissionWebhook:
+    create: true
+    # The TCP port the Webhook server binds to. (default 9448)
+    webhookBindPort: 9448
+  useCertManager: false
+  # Number of days. Only needed if useCertManager is false. (default 1)
+  certValidity: 1
 
 scheduler:
   create: true