From 810cedd1dd2151a07eda595ca2cc979373ee12d5 Mon Sep 17 00:00:00 2001 From: Ryan Warner Date: Mon, 25 Oct 2021 14:32:13 -0600 Subject: [PATCH 1/2] Track worker memory usage --- locust/runners.py | 9 +++++++-- locust/static/locust.js | 12 ++++++++++++ locust/templates/index.html | 2 ++ locust/test/test_runners.py | 27 ++++++++++++++++++++++----- locust/web.py | 1 + 5 files changed, 44 insertions(+), 7 deletions(-) diff --git a/locust/runners.py b/locust/runners.py index f4afaa4ccb..c68790ffa0 100644 --- a/locust/runners.py +++ b/locust/runners.py @@ -87,7 +87,8 @@ def __init__(self, environment): self.current_cpu_usage = 0 self.cpu_warning_emitted = False self.worker_cpu_warning_emitted = False - self.greenlet.spawn(self.monitor_cpu).link_exception(greenlet_exception_handler) + self.current_memory_usage = 0 + self.greenlet.spawn(self.monitor_cpu_and_memory).link_exception(greenlet_exception_handler) self.exceptions = {} # Because of the way the ramp-up/ramp-down is implemented, target_user_classes_count # is only updated at the end of the ramp-up/ramp-down. @@ -282,10 +283,11 @@ def stop_users(self, user_classes_stop_count: Dict[str, int]): "%g users have been stopped, %g still running", sum(user_classes_stop_count.values()), self.user_count ) - def monitor_cpu(self): + def monitor_cpu_and_memory(self): process = psutil.Process() while True: self.current_cpu_usage = process.cpu_percent() + self.current_memory_usage = process.memory_info().rss if self.current_cpu_usage > 90 and not self.cpu_warning_emitted: logging.warning( "CPU usage above 90%! This may constrain your throughput and may even give inconsistent response time measurements! See https://docs.locust.io/en/stable/running-locust-distributed.html for how to distribute the load over multiple CPU cores or machines" @@ -534,6 +536,7 @@ def __init__(self, id: str, state=STATE_INIT, heartbeat_liveness=HEARTBEAT_LIVEN self.heartbeat = heartbeat_liveness self.cpu_usage = 0 self.cpu_warning_emitted = False + self.memory_usage = 0 # The reported users running on the worker self.user_classes_count: Dict[str, int] = {} @@ -928,6 +931,7 @@ def client_listener(self): logger.warning( "Worker %s exceeded cpu threshold (will only log this once per worker)" % (msg.node_id) ) + c.memory_usage = msg.data["current_memory_usage"] elif msg.type == "stats": self.environment.events.worker_report.fire(client_id=msg.node_id, data=msg.data) elif msg.type == "spawning": @@ -1103,6 +1107,7 @@ def heartbeat(self): { "state": self.worker_state, "current_cpu_usage": self.current_cpu_usage, + "current_memory_usage": self.current_memory_usage, }, self.client_id, ) diff --git a/locust/static/locust.js b/locust/static/locust.js index a4fc9af84a..d6ef8835dc 100644 --- a/locust/static/locust.js +++ b/locust/static/locust.js @@ -45,6 +45,18 @@ $(".close_link").click(function(event) { $(this).parent().parent().hide(); }); +function formatBytes(bytes, decimals = 2) { + if (bytes === 0) return '0 Bytes'; + + const k = 1024; + const dm = decimals < 0 ? 0 : decimals; + const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']; + + const i = Math.floor(Math.log(bytes) / Math.log(k)); + + return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i]; +} + $("ul.tabs").tabs("div.panes > div").on("onClick", function (event) { // trigger resizing of charts resizeCharts(); diff --git a/locust/templates/index.html b/locust/templates/index.html index 840af1ffba..6f47260315 100644 --- a/locust/templates/index.html +++ b/locust/templates/index.html @@ -219,6 +219,7 @@

Edit running load test

State # users CPU usage + Memory usage @@ -321,6 +322,7 @@

Version <%= this.user_count %> <%= this.cpu_usage %>% + <%= formatBytes(this.memory_usage) %> <% alternate = !alternate; %> ]]> diff --git a/locust/test/test_runners.py b/locust/test/test_runners.py index 8ac4a2518c..9fb0246d88 100644 --- a/locust/test/test_runners.py +++ b/locust/test/test_runners.py @@ -1781,13 +1781,25 @@ def my_task(self): sleep(0.2) server.mocked_send( - Message("heartbeat", {"state": STATE_RUNNING, "current_cpu_usage": 50, "count": 1}, "fake_client1") + Message( + "heartbeat", + {"state": STATE_RUNNING, "current_cpu_usage": 50, "current_memory_usage": 200, "count": 1}, + "fake_client1", + ) ) server.mocked_send( - Message("heartbeat", {"state": STATE_RUNNING, "current_cpu_usage": 50, "count": 1}, "fake_client2") + Message( + "heartbeat", + {"state": STATE_RUNNING, "current_cpu_usage": 50, "current_memory_usage": 200, "count": 1}, + "fake_client2", + ) ) server.mocked_send( - Message("heartbeat", {"state": STATE_RUNNING, "current_cpu_usage": 50, "count": 1}, "fake_client3") + Message( + "heartbeat", + {"state": STATE_RUNNING, "current_cpu_usage": 50, "current_memory_usage": 200, "count": 1}, + "fake_client3", + ) ) sleep(0.2) @@ -1798,7 +1810,11 @@ def my_task(self): ) server.mocked_send( - Message("heartbeat", {"state": STATE_RUNNING, "current_cpu_usage": 50, "count": 1}, "fake_client1") + Message( + "heartbeat", + {"state": STATE_RUNNING, "current_cpu_usage": 50, "current_memory_usage": 200, "count": 1}, + "fake_client1", + ) ) sleep(0.4) @@ -2762,9 +2778,10 @@ def my_task(self): sleep(0.1) message = next((m for m in reversed(client.outbox) if m.type == "heartbeat")) - self.assertEqual(len(message.data), 2) + self.assertEqual(len(message.data), 3) self.assertIn("state", message.data) self.assertIn("current_cpu_usage", message.data) + self.assertIn("current_memory_usage", message.data) worker.quit() diff --git a/locust/web.py b/locust/web.py index 6de01e8057..bed99ed31f 100644 --- a/locust/web.py +++ b/locust/web.py @@ -308,6 +308,7 @@ def request_stats(): "state": worker.state, "user_count": worker.user_count, "cpu_usage": worker.cpu_usage, + "memory_usage": worker.memory_usage, } ) From 7de5cb669f427d666330d39460f5e37b478d793a Mon Sep 17 00:00:00 2001 From: Ryan Warner Date: Mon, 25 Oct 2021 17:14:50 -0600 Subject: [PATCH 2/2] Make memory reporting optional Don't break older workers or other language runners not updated yet. --- locust/runners.py | 3 ++- locust/static/locust.js | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/locust/runners.py b/locust/runners.py index c68790ffa0..4e33521c0d 100644 --- a/locust/runners.py +++ b/locust/runners.py @@ -931,7 +931,8 @@ def client_listener(self): logger.warning( "Worker %s exceeded cpu threshold (will only log this once per worker)" % (msg.node_id) ) - c.memory_usage = msg.data["current_memory_usage"] + if "current_memory_usage" in msg.data: + c.memory_usage = msg.data["current_memory_usage"] elif msg.type == "stats": self.environment.events.worker_report.fire(client_id=msg.node_id, data=msg.data) elif msg.type == "spawning": diff --git a/locust/static/locust.js b/locust/static/locust.js index d6ef8835dc..be0a1be256 100644 --- a/locust/static/locust.js +++ b/locust/static/locust.js @@ -47,6 +47,7 @@ $(".close_link").click(function(event) { function formatBytes(bytes, decimals = 2) { if (bytes === 0) return '0 Bytes'; + if (bytes === 0) return 'N/A'; const k = 1024; const dm = decimals < 0 ? 0 : decimals;