diff --git a/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml b/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml index 23896a230..af3525ff9 100644 --- a/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml +++ b/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml @@ -165,7 +165,7 @@ spec: {{- range $_, $mount := .Values.sidecarVolumeMount }} - name: {{ $mount.name }} mountPath: {{ $mount.mountPath }} - readOnly: true + readOnly: {{ $mount.readOnly }} {{- end }} {{- range $_, $mount := .Values.configmaps }} - name: {{ $mount.name }} diff --git a/charts/patches/kube-prometheus-stack/0000-fix-node-exporter-sidecar-volume.patch b/charts/patches/kube-prometheus-stack/0000-fix-node-exporter-sidecar-volume.patch new file mode 100644 index 000000000..f9419ec75 --- /dev/null +++ b/charts/patches/kube-prometheus-stack/0000-fix-node-exporter-sidecar-volume.patch @@ -0,0 +1,13 @@ +diff --git a/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml b/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml +index 23896a23..af3525ff 100644 +--- a/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml ++++ b/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml +@@ -165,7 +165,7 @@ spec: + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} +- readOnly: true ++ readOnly: {{ $mount.readOnly }} + {{- end }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} diff --git a/roles/kube_prometheus_stack/files/jsonnet/mixins.libsonnet b/roles/kube_prometheus_stack/files/jsonnet/mixins.libsonnet index 7c8c45eb4..f80bce0a1 100644 --- a/roles/kube_prometheus_stack/files/jsonnet/mixins.libsonnet +++ b/roles/kube_prometheus_stack/files/jsonnet/mixins.libsonnet @@ -87,6 +87,39 @@ local mixins = { }, ], }, + { + name: 'bluestore-fragmentation-score', + rules: [ + { + alert: 'BluestoreFragmentationScoreConsiderable', + annotations: { + description: 'Bluestore fragmentation score for osd {{ $labels.osd }} on host {{ $labels.instance }} is currently at {{ $value }}. If it continue to goes higher then 0.9, it will impact other running services.', + summary: '[{{ $labels.osd }}] reaching a considerable value: {{ $value }}', + }, + 'for': '1m', + expr: ||| + bluestore_allocator_score_block > 0.7 + |||, + labels: { + severity: 'warning', + }, + }, + { + alert: 'BluestoreFragmentationScoreHigh', + annotations: { + description: 'Bluestore fragmentation score for osd {{ $labels.osd }} on host {{ $labels.instance }} is currently at {{ $value }}. It might impact other running services.', + summary: '[{{ $labels.osd }}] reaching a high value: {{ $value }}', + }, + 'for': '1m', + expr: ||| + bluestore_allocator_score_block > 0.9 + |||, + labels: { + severity: 'P3', + }, + } + ], + }, ], } }, diff --git a/roles/kube_prometheus_stack/vars/main.yml b/roles/kube_prometheus_stack/vars/main.yml index 1a0be777b..761e7f7c6 100644 --- a/roles/kube_prometheus_stack/vars/main.yml +++ b/roles/kube_prometheus_stack/vars/main.yml @@ -554,7 +554,73 @@ _kube_prometheus_stack_helm_values: - --collector.processes - --collector.systemd - --collector.stat.softirq + - --collector.textfile.directory=/tmp prometheus: monitor: relabelings: *relabelings_instance_to_node_name + securityContext: + runAsNonRoot: false + sidecars: + - name: ceph-osd-bs-collector + image: "{{ atmosphere_images['ceph'] }}" + resources: + limits: + cpu: 100m + memory: 300Mi + requests: + cpu: 100m + memory: 300Mi + securityContext: + privileged: true + runAsUser: 0 + command: + - /bin/bash + - /var/lib/cron/prom-bs-frag-score-collector.sh + volumeMounts: + - name: kube-prometheus-stack-node-ceph-bs + mountPath: /var/lib/cron/prom-bs-frag-score-collector.sh + subPath: prom-bs-frag-score-collector.sh + sidecarVolumeMount: + - name: pod-tmp + mountPath: /tmp + emptyDir: {} + readOnly: false + configmaps: + - name: kube-prometheus-stack-node-ceph-bs + mountPath: /var/lib/cron + configMap: + name: kube-prometheus-stack-node-ceph-bs + defaultMode: "0755" + sidecarHostVolumeMounts: + - name: ceph-var + hostPath: /var/lib/ceph + mountPath: /var/lib/ceph + readOnly: true + - name: varrun + hostPath: /var/run + mountPath: /var/run + readOnly: true + extraManifests: + - | + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-prometheus-stack-node-ceph-bs + data: + prom-bs-frag-score-collector.sh: | + #!/bin/bash + set -ex + export ceph_mon_fsid="{{ ceph_mon_fsid }}" + export PATH="/usr/local/bin:/usr/bin:/bin:/usr/sbin" + function bs_frag_score () { + ceph-volume --log-path /tmp lvm list | grep "osd id" | grep -o [0-9]*| xargs -i sh -c 'echo bluestore_allocator_score_block{osd=\"osd.{}\"} $(ceph daemon /var/run/ceph/$ceph_mon_fsid/ceph-osd.{}.asok bluestore allocator score block | grep fragmentation_rating|cut -d ':' -f2)' + } + function collect_bs_frag_score () { + echo "Start collecting Bluestore fragmentation score" + export TMPFILE=/tmp/bluestore_allocator_score_block.prom.$$ + echo "# TYPE bluestore_allocator_score_block gauge" > $TMPFILE + bs_frag_score >> $TMPFILE + mv $TMPFILE /tmp/bluestore_allocator_score_block.prom + } + while true; do collect_bs_frag_score ; sleep 60 ; done additionalPrometheusRulesMap: "{{ lookup('vexxhost.atmosphere.jsonnet', 'jsonnet/rules.jsonnet') }}"