From cb704e7b9a137a84c5d6c37f6a7dd325cabe964e Mon Sep 17 00:00:00 2001 From: Blottiere Paul Date: Tue, 16 Apr 2024 12:53:52 +0200 Subject: [PATCH 1/4] Add qsa stats command --- qsa-api/qsa_api/api/instances.py | 13 ++++++++ qsa-api/qsa_api/monitor.py | 10 ++++++ qsa-cli/qsa/cli.py | 16 ++++++++++ qsa-plugin/__init__.py | 53 +++++++++++++++++++++++++++++++- sandbox/docker-compose.yml | 2 ++ 5 files changed, 93 insertions(+), 1 deletion(-) diff --git a/qsa-api/qsa_api/api/instances.py b/qsa-api/qsa_api/api/instances.py index 2e756ab..91d0b63 100644 --- a/qsa-api/qsa_api/api/instances.py +++ b/qsa-api/qsa_api/api/instances.py @@ -50,3 +50,16 @@ def instances_logs(instance): return {"error": "QGIS Server instance is not available"}, 415 return monitor.conns[instance].logs + + +@instances.get("//stats") +def instances_stats(instance): + monitor = current_app.config["MONITOR"] + + if not monitor: + return {"error": "QGIS Server monitoring is not activated"}, 415 + + if instance not in monitor.conns: + return {"error": "QGIS Server instance is not available"}, 415 + + return monitor.conns[instance].stats diff --git a/qsa-api/qsa_api/monitor.py b/qsa-api/qsa_api/monitor.py index f2b8c5a..ab5fb31 100644 --- a/qsa-api/qsa_api/monitor.py +++ b/qsa-api/qsa_api/monitor.py @@ -56,6 +56,16 @@ def logs(self) -> dict: print(e, file=sys.stderr) return {} + @property + def stats(self) -> dict: + self.response = None + try: + self.con.send(b"stats") + return self._wait_recv() + except Exception as e: + print(e, file=sys.stderr) + return {} + def _wait_recv(self): it = 0 diff --git a/qsa-cli/qsa/cli.py b/qsa-cli/qsa/cli.py index 0128490..c896ba2 100644 --- a/qsa-cli/qsa/cli.py +++ b/qsa-cli/qsa/cli.py @@ -2,6 +2,7 @@ import os import json +import time import click import requests from tabulate import tabulate @@ -61,3 +62,18 @@ def logs(id): data = requests.get(url) print(data.json()["logs"]) + + +@cli.command() +@click.argument("id") +def stats(id): + """ + Returns stats of a specific QGIS Server instance + """ + + while 1: + url = f"{QSA_URL}/api/instances/{id}/stats" + data = requests.get(url) + + print(data.json()) + time.sleep(0.25) diff --git a/qsa-plugin/__init__.py b/qsa-plugin/__init__.py index 8ff23ba..92731e7 100644 --- a/qsa-plugin/__init__.py +++ b/qsa-plugin/__init__.py @@ -13,11 +13,43 @@ from datetime import datetime from qgis import PyQt -from qgis.server import QgsConfigCache from qgis.utils import server_active_plugins +from qgis.server import QgsConfigCache, QgsServerFilter from qgis.core import Qgis, QgsProviderRegistry, QgsApplication LOG_MESSAGES = [] +CURRENT_TASK = {} +CURRENT_TASK_START = None + + +class ProbeFilter(QgsServerFilter): + def __init__(self, iface): + super().__init__(iface) + + def onRequestReady(self) -> bool: + print("onRequestReady", file=sys.stderr) + request = self.serverInterface().requestHandler() + params = request.parameterMap() + + CURRENT_TASK["project"] = params.get("MAP", "") + CURRENT_TASK["service"] = params.get("SERVICE", "") + CURRENT_TASK["request"] = params.get("REQUEST", "") + + CURRENT_TASK_START = datetime.now() + + return True + + def onResponseComplete(self) -> bool: + self._update() + return True + + def onSendResponse(self) -> bool: + self._update() + return True + + def _update(self) -> None: + CURRENT_TASK = {} + CURRENT_TASK_START = None def log_messages(): @@ -26,6 +58,21 @@ def log_messages(): return m +def stats(): + s = {} + s["uptime"] = 0 + s["pid"] = 0 + s["cpu"] = 0 + s["memory"] = 0 + s["task"] = {} + + if CURRENT_TASK_START is not None and CURRENT_TASK: + s["task"] = CURRENT_TASK + s["task"]["duration"] = (datetime.now() - CURRENT_TASK_START).total_seconds() * 1000 + + return s + + def metadata(iface) -> dict: m = {} m["plugins"] = server_active_plugins @@ -74,6 +121,8 @@ def f(iface, host: str, port: int) -> None: payload = metadata(iface) elif b"logs" in data: payload = log_messages() + elif b"stats" in data: + payload = stats() ser = pickle.dumps(payload) s.sendall(struct.pack(">I", len(ser))) @@ -104,3 +153,5 @@ def serverClassFactory(iface): ), ) t.start() + + iface.registerFilter(ProbeFilter(iface), 100) diff --git a/sandbox/docker-compose.yml b/sandbox/docker-compose.yml index cec4912..e529383 100644 --- a/sandbox/docker-compose.yml +++ b/sandbox/docker-compose.yml @@ -12,6 +12,8 @@ services: - QGIS_SERVER_LOG_LEVEL=0 - QSA_HOST="qsa" - QSA_PORT=9999 + ports: + - "8080:80" qsa: image: pblottiere/qsa entrypoint: "qsa" From 3bbc793562df85a6dbe870ccd1e29cf572d74d55 Mon Sep 17 00:00:00 2001 From: Blottiere Paul Date: Tue, 16 Apr 2024 17:34:49 +0200 Subject: [PATCH 2/4] Some fixes --- qsa-cli/qsa/cli.py | 60 +++++++++++++++++++++++++++++++++++++----- qsa-plugin/__init__.py | 55 ++++++++++++++++++-------------------- 2 files changed, 80 insertions(+), 35 deletions(-) diff --git a/qsa-cli/qsa/cli.py b/qsa-cli/qsa/cli.py index c896ba2..7ffb1a3 100644 --- a/qsa-cli/qsa/cli.py +++ b/qsa-cli/qsa/cli.py @@ -5,6 +5,7 @@ import time import click import requests +from pathlib import Path from tabulate import tabulate QSA_URL = os.environ.get("QSA_SERVER_URL", "http://localhost:5000/") @@ -65,15 +66,62 @@ def logs(id): @cli.command() -@click.argument("id") +@click.argument("id", required=False) def stats(id): """ Returns stats of a specific QGIS Server instance """ - while 1: - url = f"{QSA_URL}/api/instances/{id}/stats" + ids = [] + if id: + ids.append(id) + else: + url = f"{QSA_URL}/api/instances" data = requests.get(url) - - print(data.json()) - time.sleep(0.25) + for s in data.json()["servers"]: + ids.append(s["id"]) + + headers = [ + "INSTANCE ID", + "COUNT", + "TIME ", + "SERVICE", + "REQUEST", + "PROJECT", + ] + + try: + while 1: + table = [] + for i in ids: + url = f"{QSA_URL}/api/instances/{i}/stats" + task = requests.get(url).json() + + if "error" in task: + continue + + t = [] + t.append(i) + t.append(task["count"]) + + if "service" in task: + t.append(f"{task['duration']} ms") + t.append(task["service"]) + t.append(task["request"]) + p = Path(task["project"]).name + t.append(p) + else: + t.append("") + t.append("") + t.append("") + t.append("") + + table.append(t) + + s = tabulate(table, headers=headers) + os.system("cls" if os.name == "nt" else "clear") + print(s) + + time.sleep(0.25) + except: + pass diff --git a/qsa-plugin/__init__.py b/qsa-plugin/__init__.py index 92731e7..d21ceb4 100644 --- a/qsa-plugin/__init__.py +++ b/qsa-plugin/__init__.py @@ -18,38 +18,37 @@ from qgis.core import Qgis, QgsProviderRegistry, QgsApplication LOG_MESSAGES = [] -CURRENT_TASK = {} -CURRENT_TASK_START = None class ProbeFilter(QgsServerFilter): - def __init__(self, iface): + def __init__(self, iface, task): super().__init__(iface) + self.task = task def onRequestReady(self) -> bool: - print("onRequestReady", file=sys.stderr) request = self.serverInterface().requestHandler() params = request.parameterMap() - CURRENT_TASK["project"] = params.get("MAP", "") - CURRENT_TASK["service"] = params.get("SERVICE", "") - CURRENT_TASK["request"] = params.get("REQUEST", "") - - CURRENT_TASK_START = datetime.now() + self.task["project"] = params.get("MAP", "") + self.task["service"] = params.get("SERVICE", "") + self.task["request"] = params.get("REQUEST", "") + self.task["start"] = datetime.now() + self.task["count"] += 1 return True def onResponseComplete(self) -> bool: - self._update() + self._clear_task() return True def onSendResponse(self) -> bool: - self._update() + self._clear_task() return True - def _update(self) -> None: - CURRENT_TASK = {} - CURRENT_TASK_START = None + def _clear_task(self): + count = self.task["count"] + self.task.clear() + self.task["count"] = count def log_messages(): @@ -58,18 +57,12 @@ def log_messages(): return m -def stats(): - s = {} - s["uptime"] = 0 - s["pid"] = 0 - s["cpu"] = 0 - s["memory"] = 0 - s["task"] = {} - - if CURRENT_TASK_START is not None and CURRENT_TASK: - s["task"] = CURRENT_TASK - s["task"]["duration"] = (datetime.now() - CURRENT_TASK_START).total_seconds() * 1000 - +def stats(task): + s = task + if "start" in s: + s["duration"] = int( + (datetime.now() - s["start"]).total_seconds() * 1000 + ) return s @@ -108,7 +101,7 @@ def auto_connect(s: socket.socket, host: str, port: int) -> socket.socket: return s -def f(iface, host: str, port: int) -> None: +def f(iface, host: str, port: int, task: dict) -> None: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s = auto_connect(s, host, port) @@ -122,7 +115,7 @@ def f(iface, host: str, port: int) -> None: elif b"logs" in data: payload = log_messages() elif b"stats" in data: - payload = stats() + payload = stats(task) ser = pickle.dumps(payload) s.sendall(struct.pack(">I", len(ser))) @@ -144,14 +137,18 @@ def serverClassFactory(iface): host = str(os.environ.get("QSA_HOST", "localhost")) port = int(os.environ.get("QSA_PORT", 9999)) + task = {} + task["count"] = 0 + t = Thread( target=f, args=( iface, host.replace('"', ""), port, + task, ), ) t.start() - iface.registerFilter(ProbeFilter(iface), 100) + iface.registerFilter(ProbeFilter(iface, task), 100) From 5d789c9f3e97503f5edae085e9fd056b20520945 Mon Sep 17 00:00:00 2001 From: Blottiere Paul Date: Tue, 16 Apr 2024 21:28:16 +0200 Subject: [PATCH 3/4] Update doc --- docs/src/qsa-cli/commands.md | 1 + qsa-cli/qsa/cli.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/src/qsa-cli/commands.md b/docs/src/qsa-cli/commands.md index 1a0c3a7..ba87366 100644 --- a/docs/src/qsa-cli/commands.md +++ b/docs/src/qsa-cli/commands.md @@ -13,6 +13,7 @@ Commands: inspect Returns metadata about a specific QGIS Server instance logs Returns logs of a specific QGIS Server instance ps List QGIS Server instances + stats Returns stats of QGIS Server instances ```` Examples: diff --git a/qsa-cli/qsa/cli.py b/qsa-cli/qsa/cli.py index 7ffb1a3..39a9d85 100644 --- a/qsa-cli/qsa/cli.py +++ b/qsa-cli/qsa/cli.py @@ -69,7 +69,7 @@ def logs(id): @click.argument("id", required=False) def stats(id): """ - Returns stats of a specific QGIS Server instance + Returns stats of QGIS Server instances """ ids = [] From 170ee965a5b1256b6988df2ca7a6d008578e0067 Mon Sep 17 00:00:00 2001 From: Blottiere Paul Date: Tue, 16 Apr 2024 21:29:12 +0200 Subject: [PATCH 4/4] Update README --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c320c72..a050ea3 100644 --- a/README.md +++ b/README.md @@ -5,9 +5,9 @@ Components: -* [QSA REST API](https://pblottiere.github.io/QSA/qsa-api/): Flask web server with a REST API for administrating QGIS Server -* [QSA plugin](https://pblottiere.github.io/QSA/qsa-plugin/): QGIS Server plugin for introspection -* [QSA cli](https://pblottiere.github.io/QSA/qsa-cli/): Command line tool +* `qsa-api`: Flask web server with a REST API for administrating QGIS Server +* `qsa-plugin`: QGIS Server plugin for introspection +* `qsa-cli`: Command line tool Main features: * Create and manage QGIS projects stored on filesystem