[Metrics] Fix shared memory is not displayed properly (ray-project#34301

) Looks like we incorrectly recorded shared memory, and incorrectly displayed it to the metrics graph (I forgot to append ray_) Signed-off-by: Jack He <[email protected]>
ProjectsByJackHe · May 4, 2023 · c10dab0 · c10dab0
1 parent 2395e76
commit c10dab0
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 3 deletions.
diff --git a/dashboard/modules/metrics/dashboards/default_dashboard_panels.py b/dashboard/modules/metrics/dashboards/default_dashboard_panels.py
@@ -262,7 +262,7 @@ def max_plus_pending(max_resource, pending_resource):
                 legend="{{Component}}",
             ),
             Target(
-                expr="node_mem_shared_bytes{{{global_filters}}}",
+                expr="sum(ray_node_mem_shared_bytes{{{global_filters}}})",
                 legend="shared_memory",
             ),
             Target(

diff --git a/dashboard/modules/reporter/reporter_agent.py b/dashboard/modules/reporter/reporter_agent.py
@@ -869,7 +869,7 @@ def _record_stats(self, stats, cluster_stats):
         if shm_used:
             node_mem_shared = Record(
                 gauge=METRICS_GAUGES["node_mem_shared_bytes"],
-                value=mem_total,
+                value=shm_used,
                 tags={"ip": ip},
             )
             records_reported.append(node_mem_shared)

diff --git a/dashboard/modules/reporter/tests/test_reporter.py b/dashboard/modules/reporter/tests/test_reporter.py
@@ -168,7 +168,8 @@ def test_case_stats_exist():
                 "ray_node_mem_used" in metric_names,
                 "ray_node_mem_available" in metric_names,
                 "ray_node_mem_total" in metric_names,
-                "ray_component_cpu_percentage" in metric_names,
+                "ray_node_mem_total" in metric_names,
+                "ray_node_mem_shared_bytes" in metric_names,
                 "ray_component_rss_mb" in metric_names,
                 "ray_component_uss_mb" in metric_names,
                 "ray_node_disk_io_read" in metric_names,
@@ -260,6 +261,10 @@ def test_report_stats():
 
     records = agent._record_stats(STATS_TEMPLATE, cluster_stats)
     for record in records:
+        name = record.gauge.name
+        val = record.value
+        if name == "node_mem_shared_bytes":
+            assert val == STATS_TEMPLATE["shm"]
         print(record.gauge.name)
         print(record)
     assert len(records) == 33