From d24bb01a17f935b2c6e69a1c28f9ef02afbc431c Mon Sep 17 00:00:00 2001 From: Andrii Chubatiuk Date: Thu, 26 Sep 2024 22:01:59 +0300 Subject: [PATCH] vmagent: added remoteWrite param as a replacement for remoteWriteUrls --- charts/victoria-metrics-agent/CHANGELOG.md | 3 ++ charts/victoria-metrics-agent/Chart.yaml | 2 +- charts/victoria-metrics-agent/README.md | 45 ++++++++++++------- .../victoria-metrics-agent/README.md.gotmpl | 22 +++++++++ .../templates/_helpers.tpl | 44 ++++++++++++++---- .../templates/configmap.yaml | 19 ++++++++ .../templates/deployment.yaml | 15 +++++-- .../templates/statefulset.yaml | 15 +++++-- charts/victoria-metrics-agent/values.yaml | 19 ++++---- .../victoria-metrics-agent/e2e/simple.yaml | 8 +++- .../victoria-metrics-agent/lint/cluster.yaml | 6 +-- .../victoria-metrics-agent/lint/simple.yaml | 4 +- 12 files changed, 152 insertions(+), 50 deletions(-) diff --git a/charts/victoria-metrics-agent/CHANGELOG.md b/charts/victoria-metrics-agent/CHANGELOG.md index 4eaa638a3..53f1373cd 100644 --- a/charts/victoria-metrics-agent/CHANGELOG.md +++ b/charts/victoria-metrics-agent/CHANGELOG.md @@ -1,6 +1,9 @@ ## Next release +**Update note**: `remoteWriteUrls` and `multiTenantUrls` parameters were replaced by `remoteWrite`. Please follow [upgrade guide](./README.md#upgrade-to-0130) + - Fail if no remoteWriteUrls set +- Added `remoteWrite` array param, which can contain all `remoteWrite.*` flag values. Please check chart docs for details. ## 0.12.2 diff --git a/charts/victoria-metrics-agent/Chart.yaml b/charts/victoria-metrics-agent/Chart.yaml index b0d8b2938..684130540 100644 --- a/charts/victoria-metrics-agent/Chart.yaml +++ b/charts/victoria-metrics-agent/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 type: application name: victoria-metrics-agent description: Victoria Metrics Agent - collects metrics from various sources and stores them to VictoriaMetrics -version: 0.12.2 +version: 0.13.0 appVersion: v1.103.0 sources: - https://github.com/VictoriaMetrics/helm-charts diff --git a/charts/victoria-metrics-agent/README.md b/charts/victoria-metrics-agent/README.md index 3d71e22f1..a8016a64b 100644 --- a/charts/victoria-metrics-agent/README.md +++ b/charts/victoria-metrics-agent/README.md @@ -1,4 +1,4 @@ -![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![Version: 0.12.2](https://img.shields.io/badge/Version-0.12.2-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![Version: 0.13.0](https://img.shields.io/badge/Version-0.13.0-informational?style=flat-square) [![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/victoriametrics)](https://artifacthub.io/packages/helm/victoriametrics/victoria-metrics-agent) [![Slack](https://img.shields.io/badge/join%20slack-%23victoriametrics-brightgreen.svg)](https://slack.victoriametrics.com/) @@ -90,6 +90,28 @@ See the history of versions of `vma` application with command. helm history vma -n NAMESPACE ``` +## Upgrade guide + +### Upgrade to 0.13.0 + +- replace `remoteWriteUrls` to `remoteWrite`: + +Given below config + +```yaml +remoteWriteUrls: +- http://address1/api/v1/write +- http://address2/api/v1/write +``` + +should be changed to + +```yaml +remoteWrite: +- url: http://address1/api/v1/write +- url: http://address2/api/v1/write +``` + ## How to uninstall Remove application with command. @@ -882,17 +904,6 @@ name: ""

Existing secret name

- - - - multiTenantUrls - list - 
-[]
-
-
- -

Destination multitenant endpoint for scraped/received metrics

@@ -1144,14 +1155,14 @@ periodSeconds: 15 - remoteWriteUrls - list - 
-[]
+      remoteWrite
+      string
+      
+null
 
 

 
-      
WARN: need to specify at least one remote write url or one multi tenant url

+      
Generates remoteWrite.* flags and config maps with value content for values, that are of type list of map. Each item should contain url param to pass validation.

 
     
     
diff --git a/charts/victoria-metrics-agent/README.md.gotmpl b/charts/victoria-metrics-agent/README.md.gotmpl
index 6899bab29..26a26b494 100644
--- a/charts/victoria-metrics-agent/README.md.gotmpl
+++ b/charts/victoria-metrics-agent/README.md.gotmpl
@@ -10,6 +10,28 @@
 
 {{ include "chart.installSection" . }}
 
+## Upgrade guide
+
+### Upgrade to 0.13.0
+
+- replace `remoteWriteUrls` to `remoteWrite`:
+
+Given below config
+
+```yaml
+remoteWriteUrls:
+- http://address1/api/v1/write
+- http://address2/api/v1/write
+```
+
+should be changed to
+
+```yaml
+remoteWrite:
+- url: http://address1/api/v1/write
+- url: http://address2/api/v1/write
+```
+
 {{ include "chart.uninstallSection" . }}
 
 {{ include "chart.helmDocs" . }}
diff --git a/charts/victoria-metrics-agent/templates/_helpers.tpl b/charts/victoria-metrics-agent/templates/_helpers.tpl
index 7a7d8c195..ad7390a5d 100644
--- a/charts/victoria-metrics-agent/templates/_helpers.tpl
+++ b/charts/victoria-metrics-agent/templates/_helpers.tpl
@@ -78,19 +78,45 @@ Defines the name of configuration map
 
 {{- define "vmagent.args" -}}
   {{- $args := default dict -}}
-  {{- if empty .Values.remoteWriteUrls -}}
-    {{- fail "Please define at least one remoteWriteUrl" -}}
+  {{- $Values := (.helm).Values | default .Values -}}
+  {{- if empty $Values.remoteWrite -}}
+    {{- fail "Please define at least one remoteWrite" -}}
   {{- end -}}
-  {{- $_ := set $args "promscrape.config" "/config/scrape.yml" -}}
+  {{- $_ := set $args "promscrape.config" "/config/scrape/scrape.yml" -}}
   {{- $_ := set $args "remoteWrite.tmpDataPath" "/tmpData" -}}
-  {{- $_ := set $args "remoteWrite.url" .Values.remoteWriteUrls -}}
-  {{- $_ := set $args "remoteWrite.multitenantURL" .Values.multiTenantUrls -}}
+  {{- range $i, $rw := $Values.remoteWrite -}}
+    {{- if not $rw.url -}}
+      {{- fail (printf "`url` is not set for `remoteWrite` idx %d" $i) -}}
+    {{- end -}}
+    {{- range $rwKey, $rwValue := $rw -}}
+      {{- $key := printf "remoteWrite.%s" $rwKey -}}
+      {{- $param := index $args $key | default list -}}
+      {{- if or (kindIs "slice" $rwValue) (kindIs "map" $rwValue) -}}
+        {{- $param = append $param (printf "/config/rw/%d-%s.yaml" $i $rwKey) -}}
+      {{- else -}}
+        {{- $param = append $param $rwValue -}}
+      {{- end -}}
+      {{- $_ := set $args $key $param -}}
+    {{- end -}}
+  {{- end -}}
   {{- $args = mergeOverwrite $args (fromYaml (include "vm.license.flag" .)) -}}
-  {{- $args = mergeOverwrite $args .Values.extraArgs -}}
-  {{- if and .Values.statefulset.enabled .Values.statefulset.clusterMode }}
-    {{- $_ := set $args "promscrape.cluster.membersCount" .Values.replicaCount -}}
-    {{- $_ := set $args "promscrape.cluster.replicationFactor" .Values.statefulset.replicationFactor -}}
+  {{- $args = mergeOverwrite $args $Values.extraArgs -}}
+  {{- if and $Values.statefulset.enabled $Values.statefulset.clusterMode }}
+    {{- $_ := set $args "promscrape.cluster.membersCount" $Values.replicaCount -}}
+    {{- $_ := set $args "promscrape.cluster.replicationFactor" $Values.statefulset.replicationFactor -}}
     {{- $_ := set $args "promscrape.cluster.memberNum" "$(POD_NAME)" -}}
   {{- end -}}
   {{- toYaml (fromYaml (include "vm.args" $args)).args -}}
 {{- end -}}
+
+{{- define "chart.rw.config" -}}
+  {{- $rwcm := default dict }}
+  {{- range $i, $rw := .Values.remoteWrite }}
+    {{- range $rwKey, $rwValue := $rw }}
+      {{- if or (kindIs "slice" $rwValue) (kindIs "map" $rwValue) }}
+        {{- $_ := set $rwcm (printf "%d-%s.yaml" $i $rwKey) (toYaml $rwValue) }}
+      {{- end -}}
+    {{- end -}}
+  {{- end -}}
+  {{- toYaml $rwcm -}}
+{{- end -}}
diff --git a/charts/victoria-metrics-agent/templates/configmap.yaml b/charts/victoria-metrics-agent/templates/configmap.yaml
index 0f0e9872e..10fe15e30 100644
--- a/charts/victoria-metrics-agent/templates/configmap.yaml
+++ b/charts/victoria-metrics-agent/templates/configmap.yaml
@@ -1,4 +1,5 @@
 {{- if eq .Values.configMap "" }}
+---
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -15,3 +16,21 @@ data:
     {{- end }}
     {{- end }}
 {{- end }}
+{{- $rwcm := default dict }}
+{{- range $i, $rw := .Values.remoteWrite }}
+  {{- range $rwKey, $rwValue := $rw }}
+    {{- if or (kindIs "slice" $rwValue) (kindIs "map" $rwValue) }}
+      {{- $_ := set $rwcm (printf "%d-%s.yaml" $i $rwKey) (toYaml $rwValue) }}
+    {{- end -}}
+  {{- end -}}
+{{- end -}}
+{{- with (include "chart.rw.config" .) }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "chart.fullname" $ -}}-rw
+  namespace: {{ include "vm.namespace" $ }}
+  labels: {{- include "chart.labels" $ | nindent 4 }}
+data: {{ . | nindent 2 }}
+{{- end }}
diff --git a/charts/victoria-metrics-agent/templates/deployment.yaml b/charts/victoria-metrics-agent/templates/deployment.yaml
index 4fb8ebd5e..c553468e7 100644
--- a/charts/victoria-metrics-agent/templates/deployment.yaml
+++ b/charts/victoria-metrics-agent/templates/deployment.yaml
@@ -95,8 +95,12 @@ spec:
           volumeMounts:
             - name: tmpdata
               mountPath: /tmpData
-            - name: config
-              mountPath: /config
+            - name: scrape-config
+              mountPath: /config/scrape
+            {{- with (include "chart.rw.config" .) }}
+            - name: rw-config
+              mountPath: /config/rw
+            {{- end }}
             {{- range .Values.extraHostPathMounts }}
             - name: {{ .name }}
               mountPath: {{ .mountPath }}
@@ -144,7 +148,12 @@ spec:
           {{- else }}
           emptyDir: {{ toYaml .Values.emptyDir | nindent 12 }}
           {{- end }}
-        - name: config
+        {{- with (include "chart.rw.config" .) }}
+        - name: rw-config
+          configMap:
+            name: {{ include "chart.fullname" $ -}}-rw
+        {{- end }}
+        - name: scrape-config
           configMap:
             name: {{ include "chart.configname" .}}
         {{- range .Values.extraHostPathMounts }}
diff --git a/charts/victoria-metrics-agent/templates/statefulset.yaml b/charts/victoria-metrics-agent/templates/statefulset.yaml
index 7f6ca8c3e..cf82f7bdd 100644
--- a/charts/victoria-metrics-agent/templates/statefulset.yaml
+++ b/charts/victoria-metrics-agent/templates/statefulset.yaml
@@ -105,8 +105,12 @@ spec:
           volumeMounts:
             - name: tmpdata
               mountPath: /tmpData
-            - name: config
-              mountPath: /config
+            - name: scrape-config
+              mountPath: /config/scrape
+            {{- with (include "chart.rw.config" .) }}
+            - name: rw-config
+              mountPath: /config/rw
+            {{- end }}
             {{- range .Values.extraHostPathMounts }}
             - name: {{ .name }}
               mountPath: {{ .mountPath }}
@@ -156,7 +160,12 @@ spec:
           emptyDir: {{ toYaml .Values.emptyDir | nindent 12 }}
           {{- end }}
         {{- end }}
-        - name: config
+        {{- with (include "chart.rw.config" .) }}
+        - name: rw-config
+          configMap:
+            name: {{ include "chart.fullname" $ -}}-rw
+        {{- end }}
+        - name: scrape-config
           configMap:
             name: {{ include "chart.configname" . }}
         {{- range .Values.extraHostPathMounts }}
diff --git a/charts/victoria-metrics-agent/values.yaml b/charts/victoria-metrics-agent/values.yaml
index d4943133b..9e51aa014 100644
--- a/charts/victoria-metrics-agent/values.yaml
+++ b/charts/victoria-metrics-agent/values.yaml
@@ -91,16 +91,15 @@ podDisruptionBudget:
   # maxUnavailable: 1
   labels: {}
 
-# -- WARN: need to specify at least one remote write url or one multi tenant url
-remoteWriteUrls: []
-# remoteWriteUrls:
-#   - http://vm-insert:8480/insert/0/prometheus
-#   - http://prometheus:8480/insert/0/prometheus
-
-# -- Destination multitenant endpoint for scraped/received metrics
-multiTenantUrls: []
-#   - http://vm-insert-az1:8480
-#   - http://vm-insert-az2:8480
+# -- Generates `remoteWrite.*` flags and config maps with value content for values, that are of type list of map.
+# Each item should contain `url` param to pass validation.
+remoteWrite:
+# - url: http://vm-insert:8480/insert/0/prometheus
+#   urlRelabelConfig:
+#     - action: keep
+#       source_labels: [env]
+#       regex: "dev"
+# - url: http://prometheus:8480/insert/0/prometheus
 
 # -- VMAgent extra command line arguments
 extraArgs:
diff --git a/hack/helm/victoria-metrics-agent/e2e/simple.yaml b/hack/helm/victoria-metrics-agent/e2e/simple.yaml
index 9a57db9f5..bbf67e041 100644
--- a/hack/helm/victoria-metrics-agent/e2e/simple.yaml
+++ b/hack/helm/victoria-metrics-agent/e2e/simple.yaml
@@ -1,5 +1,9 @@
-remoteWriteUrls:
-  - http://cluster-victoria-metrics-cluster-vminsert.default.svc.cluster.local:8480/insert/0/prometheus/
+remoteWrite:
+  - url: http://cluster-victoria-metrics-cluster-vminsert.default.svc.cluster.local:8480/insert/0/prometheus/
+    urlRelabelConfig:
+      - action: keep
+        source_labels: [env]
+        regex: "dev"
 
 replicaCount: 3
 deployment:
diff --git a/hack/helm/victoria-metrics-agent/lint/cluster.yaml b/hack/helm/victoria-metrics-agent/lint/cluster.yaml
index e098937cf..b7ec70ce7 100644
--- a/hack/helm/victoria-metrics-agent/lint/cluster.yaml
+++ b/hack/helm/victoria-metrics-agent/lint/cluster.yaml
@@ -1,5 +1,5 @@
-remoteWriteUrls:
-  - http://cluster-victoria-metrics-cluster-vminsert.default.svc.cluster.local:8480/insert/0/prometheus/
+remoteWrite:
+  - url: http://cluster-victoria-metrics-cluster-vminsert.default.svc.cluster.local:8480/insert/0/prometheus/
 
 replicaCount: 3
 deployment:
@@ -11,4 +11,4 @@ statefulset:
 
 env:
   - name: ENV_VAR
-    value: "just make sure that env var works"
\ No newline at end of file
+    value: "just make sure that env var works"
diff --git a/hack/helm/victoria-metrics-agent/lint/simple.yaml b/hack/helm/victoria-metrics-agent/lint/simple.yaml
index b71b5363f..05237073f 100644
--- a/hack/helm/victoria-metrics-agent/lint/simple.yaml
+++ b/hack/helm/victoria-metrics-agent/lint/simple.yaml
@@ -1,5 +1,5 @@
-remoteWriteUrls:
-  - http://cluster-victoria-metrics-cluster-vminsert.default.svc.cluster.local:8480/insert/0/prometheus/
+remoteWrite:
+  - url: http://cluster-victoria-metrics-cluster-vminsert.default.svc.cluster.local:8480/insert/0/prometheus/
 
 podDisruptionBudget:
   enabled: true