Add heaptrack for milvus components (#352)

Signed-off-by: Edward Zeng <[email protected]>
milvus-io · Aug 4, 2022 · 8c0a614 · 8c0a614
1 parent 29f39e0
commit 8c0a614
Show file tree

Hide file tree

Showing 12 changed files with 308 additions and 9 deletions.
diff --git a/charts/milvus/Chart.yaml b/charts/milvus/Chart.yaml
@@ -3,7 +3,7 @@ name: milvus
 appVersion: "2.1.0"
 kubeVersion: "^1.10.0-0"
 description: Milvus is an open-source vector database built to power AI applications and vector similarity search.
-version: 3.1.2
+version: 3.1.3
 keywords:
   - milvus
   - elastic

diff --git a/charts/milvus/README.md b/charts/milvus/README.md
@@ -172,6 +172,7 @@ The following table lists the configurable parameters of the Milvus Standalone c
 | `standalone.nodeSelector`                 | Node labels for Milvus Standalone pods assignment | `{}`                                                |
 | `standalone.affinity`                     | Affinity settings for Milvus Standalone pods assignment | `{}`                                          |
 | `standalone.tolerations`                  | Toleration labels for Milvus Standalone pods assignment | `[]`                                          |
+| `standalone.heaptrack.enabled`            | Whether to enable heaptrack                             | `false`                                          |
 | `standalone.extraEnv`                     | Additional Milvus Standalone container environment variables | `[]`                                     |
 | `standalone.messageQueue`                     | Message queue for Milvus Standalone: rocksmq, pulsar, kafka | `rocksmq`                                     |
 | `standalone.rocksmq.retentionTimeInMinutes` | Set the retention time of rocksmq           | `10080`                                                 |
@@ -197,6 +198,7 @@ The following table lists the configurable parameters of the Milvus Proxy compon
 | `proxy.nodeSelector`                      | Node labels for Milvus Proxy pods assignment | `{}`                                                     |
 | `proxy.affinity`                          | Affinity settings for Milvus Proxy pods assignment | `{}`                                               |
 | `proxy.tolerations`                       | Toleration labels for Milvus Proxy pods assignment | `[]`                                               |
+| `proxy.heaptrack.enabled`            | Whether to enable heaptrack                             | `false`                                          |
 | `proxy.extraEnv`                          | Additional Milvus Proxy container environment variables | `[]`                                          |
 
 ### Milvus Root Coordinator Deployment Configuration
@@ -210,6 +212,7 @@ The following table lists the configurable parameters of the Milvus Root Coordin
 | `rootCoordinator.nodeSelector`            | Node labels for Milvus Root Coordinator pods assignment | `{}`                                          |
 | `rootCoordinator.affinity`                | Affinity settings for Milvus Root Coordinator pods assignment | `{}`                                    |
 | `rootCoordinator.tolerations`             | Toleration labels for Milvus Root Coordinator pods assignment | `[]`                                    |
+| `rootCoordinator.heaptrack.enabled`            | Whether to enable heaptrack                             | `false`                                          |
 | `rootCoordinator.extraEnv`                | Additional Milvus Root Coordinator container environment variables | `[]`                               |
 | `rootCoordinator.service.type`                       | Service type                                  | `ClusterIP`                                  |
 | `rootCoordinator.service.port`                       | Port where service is exposed                 | `19530`                                      |
@@ -232,6 +235,7 @@ The following table lists the configurable parameters of the Milvus Query Coordi
 | `queryCoordinator.nodeSelector`           | Node labels for Milvus Query Coordinator pods assignment | `{}`                                         |
 | `queryCoordinator.affinity`               | Affinity settings for Milvus Query Coordinator pods assignment | `{}`                                   |
 | `queryCoordinator.tolerations`            | Toleration labels for Milvus Query Coordinator pods assignment | `[]`                                   |
+| `queryCoordinator.heaptrack.enabled`            | Whether to enable heaptrack                             | `false`                                          |
 | `queryCoordinator.extraEnv`               | Additional Milvus Query Coordinator container environment variables | `[]`                              |
 | `queryCoordinator.service.type`                       | Service type                                  | `ClusterIP`                                 |
 | `queryCoordinator.service.port`                       | Port where service is exposed                 | `19530`                                     |
@@ -255,6 +259,7 @@ The following table lists the configurable parameters of the Milvus Query Node c
 | `queryNode.nodeSelector`                  | Node labels for Milvus Query Node pods assignment | `{}`                                                |
 | `queryNode.affinity`                      | Affinity settings for Milvus Query Node pods assignment | `{}`                                          |
 | `queryNode.tolerations`                   | Toleration labels for Milvus Query Node pods assignment | `[]`                                          |
+| `queryNode.heaptrack.enabled`            | Whether to enable heaptrack                             | `false`                                          |
 | `queryNode.extraEnv`                      | Additional Milvus Query Node container environment variables | `[]`                                     |
 
 ### Milvus Index Coordinator Deployment Configuration
@@ -268,6 +273,7 @@ The following table lists the configurable parameters of the Milvus Index Coordi
 | `indexCoordinator.nodeSelector`           | Node labels for Milvus Index Coordinator pods assignment | `{}`                                         |
 | `indexCoordinator.affinity`               | Affinity settings for Milvus Index Coordinator pods assignment | `{}`                                   |
 | `indexCoordinator.tolerations`            | Toleration labels for Milvus Index Coordinator pods assignment | `[]`                                   |
+| `indexCoordinator.heaptrack.enabled`            | Whether to enable heaptrack                             | `false`                                          |
 | `indexCoordinator.extraEnv`               | Additional Milvus Index Coordinator container environment variables | `[]`                              |
 | `indexCoordinator.service.type`                       | Service type                                  | `ClusterIP`                                 |
 | `indexCoordinator.service.port`                       | Port where service is exposed                 | `19530`                                     |
@@ -292,6 +298,7 @@ The following table lists the configurable parameters of the Milvus Index Node c
 | `indexNode.nodeSelector`                  | Node labels for Milvus Index Node pods assignment | `{}`                                                |
 | `indexNode.affinity`                      | Affinity settings for Milvus Index Node pods assignment | `{}`                                          |
 | `indexNode.tolerations`                   | Toleration labels for Milvus Index Node pods assignment | `[]`                                          |
+| `indexNode.heaptrack.enabled`            | Whether to enable heaptrack                             | `false`                                          |
 | `indexNode.extraEnv`                      | Additional Milvus Index Node container environment variables | `[]`                                     |
 | `indexNode.scheduler.buildParallel`       | Index task build paralellism | `1`                                     |
 
@@ -306,6 +313,7 @@ The following table lists the configurable parameters of the Milvus Data Coordin
 | `dataCoordinator.nodeSelector`            | Node labels for Milvus Data Coordinator pods assignment | `{}`                                          |
 | `dataCoordinator.affinity`                | Affinity settings for Milvus Data Coordinator pods assignment  | `{}`                                   |
 | `dataCoordinator.tolerations`             | Toleration labels for Milvus Data Coordinator pods assignment | `[]`                                    |
+| `dataCoordinator.heaptrack.enabled`            | Whether to enable heaptrack                             | `false`                                          |
 | `dataCoordinator.extraEnv`                | Additional Milvus Data Coordinator container environment variables | `[]`                               |
 | `dataCoordinator.service.type`                        | Service type                                  | `ClusterIP`                                 |
 | `dataCoordinator.service.port`                        | Port where service is exposed                 | `19530`                                     |
@@ -329,6 +337,7 @@ The following table lists the configurable parameters of the Milvus Data Node co
 | `dataNode.nodeSelector`                   | Node labels for Milvus Data Node pods assignment | `{}`                                                 |
 | `dataNode.affinity`                       | Affinity settings for Milvus Data Node pods assignment | `{}`                                           |
 | `dataNode.tolerations`                    | Toleration labels for Milvus Data Node pods assignment | `[]`                                           |
+| `dataNode.heaptrack.enabled`            | Whether to enable heaptrack                             | `false`                                          |
 | `dataNode.extraEnv`                       | Additional Milvus Data Node container environment variables | `[]`                                      |
 
 ### Pulsar Configuration

diff --git a/charts/milvus/templates/datacoord-deployment.yaml b/charts/milvus/templates/datacoord-deployment.yaml
@@ -34,12 +34,33 @@ spec:
         - name: {{ . }}
       {{- end }}
       {{- end }}
+      {{- if .Values.dataCoordinator.heaptrack.enabled }}
+      initContainers:
+      - name: heaptrack
+        command:
+        - /bin/bash
+        - -c
+        - "cp -r /opt/heaptrack /milvus/tools"
+        image: "{{ .Values.heaptrack.image.repository }}:{{ .Values.heaptrack.image.tag }}"
+        imagePullPolicy: {{ .Values.heaptrack.image.pullPolicy }}
+        volumeMounts:
+        - mountPath: /milvus/tools
+          name: heaptrack
+      {{- end }}
       containers:
       - name: datacoord
         image: "{{ .Values.image.all.repository }}:{{ .Values.image.all.tag }}"
         imagePullPolicy: {{ .Values.image.all.pullPolicy }}
+        {{- if .Values.dataCoordinator.heaptrack.enabled }}
+        args: ["/milvus/tools/heaptrack/bin/heaptrack", "milvus", "run", "datacoord"]
+        {{- else }}
         args: [ "milvus", "run", "datacoord" ]
+        {{- end }}
         env:
+        {{- if .Values.dataCoordinator.heaptrack.enabled }}
+        - name: LD_LIBRARY_PATH
+          value: /milvus/tools/heaptrack/lib:/milvus/lib:/usr/lib
+        {{- end }}
         - name: CACHE_SIZE
           valueFrom:
             resourceFieldRef:
@@ -89,6 +110,10 @@ spec:
           mountPath: {{ .Values.log.persistence.mountPath | quote }}
           subPath: {{ .Values.log.persistence.persistentVolumeClaim.subPath | default "" }}
         {{- end }}
+        {{- if .Values.dataCoordinator.heaptrack.enabled }}
+        - mountPath: /milvus/tools
+          name: heaptrack
+        {{- end }}
 
     {{- if and (.Values.nodeSelector) (not .Values.dataCoordinator.nodeSelector) }}
       nodeSelector:
@@ -124,4 +149,8 @@ spec:
         persistentVolumeClaim:
           claimName: {{ .Values.log.persistence.persistentVolumeClaim.existingClaim | default (printf "%s-logs" (include "milvus.fullname" . | trunc 58)) }}
       {{- end }}
+      {{- if .Values.dataCoordinator.heaptrack.enabled }}
+      - name: heaptrack
+        emptyDir: {}
+      {{- end }}
 {{- end }}
diff --git a/charts/milvus/templates/datanode-deployment.yaml b/charts/milvus/templates/datanode-deployment.yaml
@@ -33,12 +33,33 @@ spec:
         - name: {{ . }}
       {{- end }}
       {{- end }}
+      {{- if .Values.dataNode.heaptrack.enabled }}
+      initContainers:
+      - name: heaptrack
+        command:
+        - /bin/bash
+        - -c
+        - "cp -r /opt/heaptrack /milvus/tools"
+        image: "{{ .Values.heaptrack.image.repository }}:{{ .Values.heaptrack.image.tag }}"
+        imagePullPolicy: {{ .Values.heaptrack.image.pullPolicy }}
+        volumeMounts:
+        - mountPath: /milvus/tools
+          name: heaptrack
+      {{- end }}
       containers:
       - name: datanode
         image: "{{ .Values.image.all.repository }}:{{ .Values.image.all.tag }}"
         imagePullPolicy: {{ .Values.image.all.pullPolicy }}
+        {{- if .Values.dataNode.heaptrack.enabled }}
+        args: [ "/milvus/tools/heaptrack/bin/heaptrack", "milvus", "run", "datanode" ]
+        {{- else }}
         args: [ "milvus", "run", "datanode" ]
+        {{- end }}
         env:
+        {{- if .Values.dataNode.heaptrack.enabled }}
+        - name: LD_LIBRARY_PATH
+          value: /milvus/tools/heaptrack/lib:/milvus/lib:/usr/lib
+        {{- end }}
         - name: CACHE_SIZE
           valueFrom:
             resourceFieldRef:
@@ -88,6 +109,10 @@ spec:
           mountPath: {{ .Values.log.persistence.mountPath | quote }}
           subPath: {{ .Values.log.persistence.persistentVolumeClaim.subPath | default "" }}
         {{- end }}
+        {{- if .Values.dataNode.heaptrack.enabled }}
+        - mountPath: /milvus/tools
+          name: heaptrack
+        {{- end }}
 
     {{- if and (.Values.nodeSelector) (not .Values.dataNode.nodeSelector) }}
       nodeSelector:
@@ -123,4 +148,8 @@ spec:
         persistentVolumeClaim:
           claimName: {{ .Values.log.persistence.persistentVolumeClaim.existingClaim | default (printf "%s-logs" (include "milvus.fullname" . | trunc 58)) }}
       {{- end }}
+      {{- if .Values.dataNode.heaptrack.enabled }}
+      - name: heaptrack
+        emptyDir: {}
+      {{- end }}
 {{- end }}
diff --git a/charts/milvus/templates/indexcoord-deployment.yaml b/charts/milvus/templates/indexcoord-deployment.yaml
@@ -34,12 +34,33 @@ spec:
         - name: {{ . }}
       {{- end }}
       {{- end }}
+      {{- if .Values.indexCoordinator.heaptrack.enabled }}
+      initContainers:
+      - name: heaptrack
+        command:
+        - /bin/bash
+        - -c
+        - "cp -r /opt/heaptrack /milvus/tools"
+        image: "{{ .Values.heaptrack.image.repository }}:{{ .Values.heaptrack.image.tag }}"
+        imagePullPolicy: {{ .Values.heaptrack.image.pullPolicy }}
+        volumeMounts:
+        - mountPath: /milvus/tools
+          name: heaptrack
+      {{- end }}
       containers:
       - name: indexcoord
         image: "{{ .Values.image.all.repository }}:{{ .Values.image.all.tag }}"
         imagePullPolicy: {{ .Values.image.all.pullPolicy }}
+        {{- if .Values.indexCoordinator.heaptrack.enabled }}
+        args: [ "/milvus/tools/heaptrack/bin/heaptrack", "milvus", "run", "indexcoord" ]
+        {{- else }}
         args: [ "milvus", "run", "indexcoord" ]
+        {{- end }}
         env:
+        {{- if .Values.indexCoordinator.heaptrack.enabled }}
+        - name: LD_LIBRARY_PATH
+          value: /milvus/tools/heaptrack/lib:/milvus/lib:/usr/lib
+        {{- end }}
         - name: CACHE_SIZE
           valueFrom:
             resourceFieldRef:
@@ -89,6 +110,10 @@ spec:
           mountPath: {{ .Values.log.persistence.mountPath | quote }}
           subPath: {{ .Values.log.persistence.persistentVolumeClaim.subPath | default "" }}
         {{- end }}
+        {{- if .Values.indexCoordinator.heaptrack.enabled }}
+        - mountPath: /milvus/tools
+          name: heaptrack
+        {{- end }}
 
     {{- if and (.Values.nodeSelector) (not .Values.indexCoordinator.nodeSelector) }}
       nodeSelector:
@@ -124,4 +149,8 @@ spec:
         persistentVolumeClaim:
           claimName: {{ .Values.log.persistence.persistentVolumeClaim.existingClaim | default (printf "%s-logs" (include "milvus.fullname" . | trunc 58)) }}
       {{- end }}
+      {{- if .Values.indexCoordinator.heaptrack.enabled }}
+      - name: heaptrack
+        emptyDir: {}
+      {{- end }}
 {{- end }}
diff --git a/charts/milvus/templates/indexnode-deployment.yaml b/charts/milvus/templates/indexnode-deployment.yaml
@@ -32,12 +32,33 @@ spec:
         - name: {{ . }}
       {{- end }}
       {{- end }}
+      {{- if .Values.indexNode.heaptrack.enabled }}
+      initContainers:
+      - name: heaptrack
+        command:
+        - /bin/bash
+        - -c
+        - "cp -r /opt/heaptrack /milvus/tools"
+        image: "{{ .Values.heaptrack.image.repository }}:{{ .Values.heaptrack.image.tag }}"
+        imagePullPolicy: {{ .Values.heaptrack.image.pullPolicy }}
+        volumeMounts:
+        - mountPath: /milvus/tools
+          name: heaptrack
+      {{- end }}
       containers:
       - name: indexnode
         image: "{{ .Values.image.all.repository }}:{{ .Values.image.all.tag }}"
         imagePullPolicy: {{ .Values.image.all.pullPolicy }}
+        {{- if .Values.indexNode.heaptrack.enabled }}
+        args: [ "/milvus/tools/heaptrack/bin/heaptrack", "milvus", "run", "indexnode" ]
+        {{- else }}
         args: [ "milvus", "run", "indexnode" ]
+        {{- end }}
         env:
+        {{- if .Values.indexNode.heaptrack.enabled }}
+        - name: LD_LIBRARY_PATH
+          value: /milvus/tools/heaptrack/lib:/milvus/lib:/usr/lib
+        {{- end }}
         - name: CACHE_SIZE
           valueFrom:
             resourceFieldRef:
@@ -87,6 +108,10 @@ spec:
           mountPath: {{ .Values.log.persistence.mountPath | quote }}
           subPath: {{ .Values.log.persistence.persistentVolumeClaim.subPath | default "" }}
         {{- end }}
+        {{- if .Values.indexNode.heaptrack.enabled }}
+        - mountPath: /milvus/tools
+          name: heaptrack
+        {{- end }}
 
 
     {{- if and (.Values.nodeSelector) (not .Values.indexNode.nodeSelector) }}
@@ -123,4 +148,8 @@ spec:
         persistentVolumeClaim:
           claimName: {{ .Values.log.persistence.persistentVolumeClaim.existingClaim | default (printf "%s-logs" (include "milvus.fullname" . | trunc 58)) }}
       {{- end }}
+      {{- if .Values.indexNode.heaptrack.enabled }}
+      - name: heaptrack
+        emptyDir: {}
+      {{- end }}
 {{- end }}