Change: cleanup, restructure, added prometheus-rule(s) #30

- add prometheus-rule for flux reconcilation stuck
- add prometheus-rule for flux reconcilation suspended
- removed not used manifests

cluster/apps/flux-system/gotk/helm-controller/kustomization.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - pod-monitor.yaml

cluster/apps/observability/kube-prometheus-stack/pod-monitors/flux-system/helm-controller.yaml → cluster/apps/flux-system/gotk/helm-controller/pod-monitor.yaml

@@ -8,13 +8,9 @@ metadata:
     app.kubernetes.io/part-of: flux
 spec:
   namespaceSelector:
-    matchNames:
-      - flux-system
+    matchNames: ["flux-system"]
   selector:
-    matchExpressions:
-      - key: app
-        operator: In
-        values:
-          - helm-controller
+    matchLabels:
+      app: helm-controller
   podMetricsEndpoints:
     - port: http-prom

cluster/apps/flux-system/gotk/kustomization.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - notification-controller
+  - helm-controller
+  - kustomize-controller
+  - source-controller
+  - prometheus-rule.yaml

cluster/apps/flux-system/gotk/kustomize-controller/kustomization.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - pod-monitor.yaml

cluster/apps/observability/kube-prometheus-stack/pod-monitors/flux-system/kustomize-controller.yaml → cluster/apps/flux-system/gotk/kustomize-controller/pod-monitor.yaml

@@ -8,13 +8,9 @@ metadata:
     app.kubernetes.io/part-of: flux
 spec:
   namespaceSelector:
-    matchNames:
-      - flux-system
+    matchNames: ["flux-system"]
   selector:
-    matchExpressions:
-      - key: app
-        operator: In
-        values:
-          - kustomize-controller
+    matchLabels:
+      app: kustomize-controller
   podMetricsEndpoints:
     - port: http-prom

cluster/apps/flux-system/notifications/kustomization.yaml → cluster/apps/flux-system/gotk/notification-controller/kustomization.yaml

@@ -5,3 +5,4 @@ resources:
   - flux-notifications-webhook-url.sops.yaml
   - alert.yaml
   - provider.yaml
+  - pod-monitor.yaml

cluster/apps/observability/kube-prometheus-stack/pod-monitors/flux-system/notification-controller.yaml → cluster/apps/flux-system/gotk/notification-controller/pod-monitor.yaml

@@ -8,13 +8,9 @@ metadata:
     app.kubernetes.io/part-of: flux
 spec:
   namespaceSelector:
-    matchNames:
-      - flux-system
+    matchNames: ["flux-system"]
   selector:
-    matchExpressions:
-      - key: app
-        operator: In
-        values:
-          - notification-controller
+    matchLabels:
+      app: notification-controller
   podMetricsEndpoints:
     - port: http-prom

cluster/apps/flux-system/gotk/prometheus-rule.yaml

@@ -0,0 +1,28 @@
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  labels:
+    app.kubernetes.io/name: flux
+    app.kubernetes.io/instance: flux
+    app.kubernetes.io/managed-by: fluxcd
+  name: flux-rules
+  namespace: flux-system
+spec:
+  groups:
+    - name: flux.rules
+      rules:
+        - alert: ReconciliationFailure
+          expr: max(gotk_reconcile_condition{status="False",type="Ready"}) by (exported_namespace, name, kind) + on(exported_namespace, name, kind) (max(gotk_reconcile_condition{status="Deleted"}) by (exported_namespace, name, kind)) * 2 == 1
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: '{{ $labels.kind }} {{ $labels.exported_namespace }}/{{ $labels.name }} reconciliation has been failing for more than ten minutes.'
+        - alert: ReconcilatonSuspended
+          expr: max(gotk_suspend_status) by (exported_namespace, kind, name) >= 1
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: '{{ $labels.kind }} {{ $labels.exported_namespace }}/{{ $labels.name }} reconciliation has been suspended for more than fifteen minutes.'

cluster/apps/flux-system/gotk/source-controller/kustomization.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - pod-monitor.yaml

cluster/apps/observability/kube-prometheus-stack/pod-monitors/flux-system/source-controller.yaml → cluster/apps/flux-system/gotk/source-controller/pod-monitor.yaml

@@ -8,13 +8,9 @@ metadata:
     app.kubernetes.io/part-of: flux
 spec:
   namespaceSelector:
-    matchNames:
-      - flux-system
+    matchNames: ["flux-system"]
   selector:
-    matchExpressions:
-      - key: app
-        operator: In
-        values:
-          - source-controller
+    matchLabels:
+      app: source-controller
   podMetricsEndpoints:
     - port: http-prom

cluster/apps/kustomization.yaml

@@ -2,14 +2,14 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - flux-system/notifications
-  - hardware/pimoroni-fanshim
-  - hardware/scanservjs
+  - flux-system/gotk
   - kube-system/metrics-server
   - kube-system/node-feature-discovery
   - kube-system/reflector
   - kube-system/reloader
   - kube-system/kured
+  - hardware/pimoroni-fanshim
+  - hardware/scanservjs
   - longhorn-system/longhorn
   - observability/loki
   - observability/promtail

cluster/apps/observability/kube-prometheus-stack/kustomization.yaml

@@ -3,8 +3,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - helm-release.yaml
-  - prometheus-rules/node-exporter.yaml
-  - pod-monitors
+  - node-exporter/prometheus-rule.yaml
 namespace: observability
 configurations:
   - kustomize-config.yaml