vmagent: added remoteWrite param as a replacement for remoteWriteUrls

VictoriaMetrics · Sep 26, 2024 · d24bb01 · d24bb01
1 parent 5cc8c6f
commit d24bb01
Show file tree

Hide file tree

Showing 12 changed files with 152 additions and 50 deletions.
diff --git a/charts/victoria-metrics-agent/CHANGELOG.md b/charts/victoria-metrics-agent/CHANGELOG.md
@@ -1,6 +1,9 @@
 ## Next release
 
+**Update note**: `remoteWriteUrls` and `multiTenantUrls` parameters were replaced by `remoteWrite`. Please follow [upgrade guide](./README.md#upgrade-to-0130)
+
 - Fail if no remoteWriteUrls set
+- Added `remoteWrite` array param, which can contain all `remoteWrite.*` flag values. Please check chart docs for details.
 
 ## 0.12.2
 

diff --git a/charts/victoria-metrics-agent/Chart.yaml b/charts/victoria-metrics-agent/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 type: application
 name: victoria-metrics-agent
 description: Victoria Metrics Agent - collects metrics from various sources and stores them to VictoriaMetrics
-version: 0.12.2
+version: 0.13.0
 appVersion: v1.103.0
 sources:
   - https://github.com/VictoriaMetrics/helm-charts

diff --git a/charts/victoria-metrics-agent/README.md b/charts/victoria-metrics-agent/README.md
@@ -1,4 +1,4 @@
-![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![Version: 0.12.2](https://img.shields.io/badge/Version-0.12.2-informational?style=flat-square)
+![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![Version: 0.13.0](https://img.shields.io/badge/Version-0.13.0-informational?style=flat-square)
 [![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/victoriametrics)](https://artifacthub.io/packages/helm/victoriametrics/victoria-metrics-agent)
 [![Slack](https://img.shields.io/badge/join%20slack-%23victoriametrics-brightgreen.svg)](https://slack.victoriametrics.com/)
 
@@ -90,6 +90,28 @@ See the history of versions of `vma` application with command.
 helm history vma -n NAMESPACE
 ```
 
+## Upgrade guide
+
+### Upgrade to 0.13.0
+
+- replace `remoteWriteUrls` to `remoteWrite`:
+
+Given below config
+
+```yaml
+remoteWriteUrls:
+- http://address1/api/v1/write
+- http://address2/api/v1/write
+```
+
+should be changed to
+
+```yaml
+remoteWrite:
+- url: http://address1/api/v1/write
+- url: http://address2/api/v1/write
+```
+
 ## How to uninstall
 
 Remove application with command.
@@ -882,17 +904,6 @@ name: ""
 </pre>
 </td>
       <td><p>Existing secret name</p>
-</td>
-    </tr>
-    <tr>
-      <td>multiTenantUrls</td>
-      <td>list</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
-<code class="language-yaml">[]
-</code>
-</pre>
-</td>
-      <td><p>Destination multitenant endpoint for scraped/received metrics</p>
 </td>
     </tr>
     <tr>
@@ -1144,14 +1155,14 @@ periodSeconds: 15
 </td>
     </tr>
     <tr>
-      <td>remoteWriteUrls</td>
-      <td>list</td>
-      <td><pre class="helm-vars-default-value" language-yaml" lang="plaintext">
-<code class="language-yaml">[]
+      <td>remoteWrite</td>
+      <td>string</td>
+      <td><pre class="helm-vars-default-value" language-yaml" lang="">
+<code class="language-yaml">null
 </code>
 </pre>
 </td>
-      <td><p>WARN: need to specify at least one remote write url or one multi tenant url</p>
+      <td><p>Generates <code>remoteWrite.*</code> flags and config maps with value content for values, that are of type list of map. Each item should contain <code>url</code> param to pass validation.</p>
 </td>
     </tr>
     <tr>

diff --git a/charts/victoria-metrics-agent/README.md.gotmpl b/charts/victoria-metrics-agent/README.md.gotmpl
@@ -10,6 +10,28 @@
 
 {{ include "chart.installSection" . }}
 
+## Upgrade guide
+
+### Upgrade to 0.13.0
+
+- replace `remoteWriteUrls` to `remoteWrite`:
+
+Given below config
+
+```yaml
+remoteWriteUrls:
+- http://address1/api/v1/write
+- http://address2/api/v1/write
+```
+
+should be changed to
+
+```yaml
+remoteWrite:
+- url: http://address1/api/v1/write
+- url: http://address2/api/v1/write
+```
+
 {{ include "chart.uninstallSection" . }}
 
 {{ include "chart.helmDocs" . }}

diff --git a/charts/victoria-metrics-agent/templates/_helpers.tpl b/charts/victoria-metrics-agent/templates/_helpers.tpl
@@ -78,19 +78,45 @@ Defines the name of configuration map
 
 {{- define "vmagent.args" -}}
   {{- $args := default dict -}}
-  {{- if empty .Values.remoteWriteUrls -}}
-    {{- fail "Please define at least one remoteWriteUrl" -}}
+  {{- $Values := (.helm).Values | default .Values -}}
+  {{- if empty $Values.remoteWrite -}}
+    {{- fail "Please define at least one remoteWrite" -}}
   {{- end -}}
-  {{- $_ := set $args "promscrape.config" "/config/scrape.yml" -}}
+  {{- $_ := set $args "promscrape.config" "/config/scrape/scrape.yml" -}}
   {{- $_ := set $args "remoteWrite.tmpDataPath" "/tmpData" -}}
-  {{- $_ := set $args "remoteWrite.url" .Values.remoteWriteUrls -}}
-  {{- $_ := set $args "remoteWrite.multitenantURL" .Values.multiTenantUrls -}}
+  {{- range $i, $rw := $Values.remoteWrite -}}
+    {{- if not $rw.url -}}
+      {{- fail (printf "`url` is not set for `remoteWrite` idx %d" $i) -}}
+    {{- end -}}
+    {{- range $rwKey, $rwValue := $rw -}}
+      {{- $key := printf "remoteWrite.%s" $rwKey -}}
+      {{- $param := index $args $key | default list -}}
+      {{- if or (kindIs "slice" $rwValue) (kindIs "map" $rwValue) -}}
+        {{- $param = append $param (printf "/config/rw/%d-%s.yaml" $i $rwKey) -}}
+      {{- else -}}
+        {{- $param = append $param $rwValue -}}
+      {{- end -}}
+      {{- $_ := set $args $key $param -}}
+    {{- end -}}
+  {{- end -}}
   {{- $args = mergeOverwrite $args (fromYaml (include "vm.license.flag" .)) -}}
-  {{- $args = mergeOverwrite $args .Values.extraArgs -}}
-  {{- if and .Values.statefulset.enabled .Values.statefulset.clusterMode }}
-    {{- $_ := set $args "promscrape.cluster.membersCount" .Values.replicaCount -}}
-    {{- $_ := set $args "promscrape.cluster.replicationFactor" .Values.statefulset.replicationFactor -}}
+  {{- $args = mergeOverwrite $args $Values.extraArgs -}}
+  {{- if and $Values.statefulset.enabled $Values.statefulset.clusterMode }}
+    {{- $_ := set $args "promscrape.cluster.membersCount" $Values.replicaCount -}}
+    {{- $_ := set $args "promscrape.cluster.replicationFactor" $Values.statefulset.replicationFactor -}}
     {{- $_ := set $args "promscrape.cluster.memberNum" "$(POD_NAME)" -}}
   {{- end -}}
   {{- toYaml (fromYaml (include "vm.args" $args)).args -}}
 {{- end -}}
+
+{{- define "chart.rw.config" -}}
+  {{- $rwcm := default dict }}
+  {{- range $i, $rw := .Values.remoteWrite }}
+    {{- range $rwKey, $rwValue := $rw }}
+      {{- if or (kindIs "slice" $rwValue) (kindIs "map" $rwValue) }}
+        {{- $_ := set $rwcm (printf "%d-%s.yaml" $i $rwKey) (toYaml $rwValue) }}
+      {{- end -}}
+    {{- end -}}
+  {{- end -}}
+  {{- toYaml $rwcm -}}
+{{- end -}}
diff --git a/charts/victoria-metrics-agent/templates/configmap.yaml b/charts/victoria-metrics-agent/templates/configmap.yaml
@@ -1,4 +1,5 @@
 {{- if eq .Values.configMap "" }}
+---
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -15,3 +16,21 @@ data:
     {{- end }}
     {{- end }}
 {{- end }}
+{{- $rwcm := default dict }}
+{{- range $i, $rw := .Values.remoteWrite }}
+  {{- range $rwKey, $rwValue := $rw }}
+    {{- if or (kindIs "slice" $rwValue) (kindIs "map" $rwValue) }}
+      {{- $_ := set $rwcm (printf "%d-%s.yaml" $i $rwKey) (toYaml $rwValue) }}
+    {{- end -}}
+  {{- end -}}
+{{- end -}}
+{{- with (include "chart.rw.config" .) }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "chart.fullname" $ -}}-rw
+  namespace: {{ include "vm.namespace" $ }}
+  labels: {{- include "chart.labels" $ | nindent 4 }}
+data: {{ . | nindent 2 }}
+{{- end }}
diff --git a/charts/victoria-metrics-agent/templates/deployment.yaml b/charts/victoria-metrics-agent/templates/deployment.yaml
@@ -95,8 +95,12 @@ spec:
           volumeMounts:
             - name: tmpdata
               mountPath: /tmpData
-            - name: config
-              mountPath: /config
+            - name: scrape-config
+              mountPath: /config/scrape
+            {{- with (include "chart.rw.config" .) }}
+            - name: rw-config
+              mountPath: /config/rw
+            {{- end }}
             {{- range .Values.extraHostPathMounts }}
             - name: {{ .name }}
               mountPath: {{ .mountPath }}
@@ -144,7 +148,12 @@ spec:
           {{- else }}
           emptyDir: {{ toYaml .Values.emptyDir | nindent 12 }}
           {{- end }}
-        - name: config
+        {{- with (include "chart.rw.config" .) }}
+        - name: rw-config
+          configMap:
+            name: {{ include "chart.fullname" $ -}}-rw
+        {{- end }}
+        - name: scrape-config
           configMap:
             name: {{ include "chart.configname" .}}
         {{- range .Values.extraHostPathMounts }}

diff --git a/charts/victoria-metrics-agent/templates/statefulset.yaml b/charts/victoria-metrics-agent/templates/statefulset.yaml
@@ -105,8 +105,12 @@ spec:
           volumeMounts:
             - name: tmpdata
               mountPath: /tmpData
-            - name: config
-              mountPath: /config
+            - name: scrape-config
+              mountPath: /config/scrape
+            {{- with (include "chart.rw.config" .) }}
+            - name: rw-config
+              mountPath: /config/rw
+            {{- end }}
             {{- range .Values.extraHostPathMounts }}
             - name: {{ .name }}
               mountPath: {{ .mountPath }}
@@ -156,7 +160,12 @@ spec:
           emptyDir: {{ toYaml .Values.emptyDir | nindent 12 }}
           {{- end }}
         {{- end }}
-        - name: config
+        {{- with (include "chart.rw.config" .) }}
+        - name: rw-config
+          configMap:
+            name: {{ include "chart.fullname" $ -}}-rw
+        {{- end }}
+        - name: scrape-config
           configMap:
             name: {{ include "chart.configname" . }}
         {{- range .Values.extraHostPathMounts }}

diff --git a/charts/victoria-metrics-agent/values.yaml b/charts/victoria-metrics-agent/values.yaml
@@ -91,16 +91,15 @@ podDisruptionBudget:
   # maxUnavailable: 1
   labels: {}
 
-# -- WARN: need to specify at least one remote write url or one multi tenant url
-remoteWriteUrls: []
-# remoteWriteUrls:
-#   - http://vm-insert:8480/insert/0/prometheus
-#   - http://prometheus:8480/insert/0/prometheus
-
-# -- Destination multitenant endpoint for scraped/received metrics
-multiTenantUrls: []
-#   - http://vm-insert-az1:8480
-#   - http://vm-insert-az2:8480
+# -- Generates `remoteWrite.*` flags and config maps with value content for values, that are of type list of map.
+# Each item should contain `url` param to pass validation.
+remoteWrite:
+# - url: http://vm-insert:8480/insert/0/prometheus
+#   urlRelabelConfig:
+#     - action: keep
+#       source_labels: [env]
+#       regex: "dev"
+# - url: http://prometheus:8480/insert/0/prometheus
 
 # -- VMAgent extra command line arguments
 extraArgs:

diff --git a/hack/helm/victoria-metrics-agent/e2e/simple.yaml b/hack/helm/victoria-metrics-agent/e2e/simple.yaml
@@ -1,5 +1,9 @@
-remoteWriteUrls:
-  - http://cluster-victoria-metrics-cluster-vminsert.default.svc.cluster.local:8480/insert/0/prometheus/
+remoteWrite:
+  - url: http://cluster-victoria-metrics-cluster-vminsert.default.svc.cluster.local:8480/insert/0/prometheus/
+    urlRelabelConfig:
+      - action: keep
+        source_labels: [env]
+        regex: "dev"
 
 replicaCount: 3
 deployment:

diff --git a/hack/helm/victoria-metrics-agent/lint/cluster.yaml b/hack/helm/victoria-metrics-agent/lint/cluster.yaml
@@ -1,5 +1,5 @@
-remoteWriteUrls:
-  - http://cluster-victoria-metrics-cluster-vminsert.default.svc.cluster.local:8480/insert/0/prometheus/
+remoteWrite:
+  - url: http://cluster-victoria-metrics-cluster-vminsert.default.svc.cluster.local:8480/insert/0/prometheus/
 
 replicaCount: 3
 deployment:
@@ -11,4 +11,4 @@ statefulset:
 
 env:
   - name: ENV_VAR
-    value: "just make sure that env var works"
+    value: "just make sure that env var works"
diff --git a/hack/helm/victoria-metrics-agent/lint/simple.yaml b/hack/helm/victoria-metrics-agent/lint/simple.yaml
@@ -1,5 +1,5 @@
-remoteWriteUrls:
-  - http://cluster-victoria-metrics-cluster-vminsert.default.svc.cluster.local:8480/insert/0/prometheus/
+remoteWrite:
+  - url: http://cluster-victoria-metrics-cluster-vminsert.default.svc.cluster.local:8480/insert/0/prometheus/
 
 podDisruptionBudget:
   enabled: true