Merge branch 'feature/add-health-checks' into 'devel'

Add redis and vault monitoring to /health endpoint

Closes #1047

See merge request sds-dev/sd-connect/swift-browser-ui!93

CHANGELOG.md

@@ -68,6 +68,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - (GL #923) Add `Last modified` column for Container Table
 - (GL #999) Add three level sharing with view option
 - (GL #947) Added cypress tests for login, adding and showing containers, uploading files
+- (GL #1047) Added redis and vault monitoring to `/health` endpoint
 
 ### Changed
 

swift_browser_ui/common/vault_client.py

@@ -394,3 +394,13 @@ async def put_header(
                 f"c4ghtransit/files/{project}/{container}/{path}",
                 json_data={"header": header},
             )
+
+    async def get_sys_health(self) -> str:
+        """Check Vault client health."""
+        resp = await self._request("GET", "sys/health", timeout=5)
+        if isinstance(resp, Dict):
+            data: Dict = resp
+        if data["initialized"] and data["sealed"] is False:
+            return "Ok"
+        else:
+            return "Down"

swift_browser_ui/ui/_convenience.py

@@ -6,6 +6,7 @@
 
 
 import logging
+import os
 import secrets
 import ssl
 import typing
@@ -14,7 +15,9 @@
 import aiohttp.web
 import aiohttp_session
 import certifi
+import redis.asyncio as redis
 import requests
+from redis.asyncio.sentinel import Sentinel
 
 import swift_browser_ui.common.signature
 from swift_browser_ui.ui.settings import setd
@@ -191,3 +194,33 @@ async def open_upload_runner_session(
             session["projects"][project]["runner"] = ret
             session.changed()
         return ret
+
+
+async def get_redis_client() -> redis.Redis:
+    """Initialize and return a Python Redis client."""
+    sentinel_url = str(os.environ.get("SWIFT_UI_REDIS_SENTINEL_HOST", ""))
+    sentinel_port = str(os.environ.get("SWIFT_UI_REDIS_SENTINEL_PORT", ""))
+    sentinel_master = os.environ.get("SWIFT_UI_REDIS_SENTINEL_MASTER", "mymaster")
+
+    redis_user = str(os.environ.get("SWIFT_UI_REDIS_USER", ""))
+    redis_password = str(os.environ.get("SWIFT_UI_REDIS_PASSWORD", ""))
+
+    if sentinel_url and sentinel_port:
+        # Auth is forwarded to redis so no need for auth on sentinel
+        sentinel = Sentinel([(str(sentinel_url), int(sentinel_port))])
+
+        redis_client = sentinel.master_for(
+            service_name=sentinel_master,
+            redis_class=redis.Redis,
+            password=redis_password,
+            username=redis_user,
+        )
+    else:
+        redis_port = str(os.environ.get("SWIFT_UI_REDIS_PORT", ""))
+        redis_host = str(os.environ.get("SWIFT_UI_REDIS_HOST", "localhost"))
+
+        redis_creds = ""
+        if redis_user and redis_password:
+            redis_creds = f"{redis_user}:{redis_password}@"
+        redis_client = redis.from_url(f"redis://{redis_creds}{redis_host}:{redis_port}")
+    return redis_client

swift_browser_ui/ui/health.py

@@ -6,8 +6,10 @@
 
 import aiohttp.web
 from aiohttp.client_exceptions import ServerDisconnectedError
+from redis import ConnectionError
 
 import swift_browser_ui.common.signature
+from swift_browser_ui.ui._convenience import get_redis_client
 from swift_browser_ui.ui.settings import setd
 
 
@@ -100,20 +102,51 @@ async def get_upload_runner(
             ) as resp:
                 request.app["Log"].debug(resp)
                 if resp.status != 200:
-                    services["swiftui-upload-runner"] = {
-                        "status": "Down",
-                    }
+                    services["swiftui-upload-runner"] = {"status": "Down"}
+                    services["vault"] = {"status": "Down"}
+                    end = time.time() - start
+                    performance["swiftui-upload-runner"] = {"time": end}
+                    performance["vault"] = {"time": end}
                 else:
-                    upload_status = await resp.json()
-                    services["swiftui-upload-runner"] = upload_status
-            performance["swiftui-upload-runner"] = {"time": time.time() - start}
+                    status = await resp.json()
+                    services["swiftui-upload-runner"] = status["upload-runner"]
+                    services["vault"] = status["vault-instance"]
+                    performance["swiftui-upload-runner"] = {
+                        "time": status["start-time"] - start
+                    }
+                    performance["vault"] = {
+                        "time": status["end-time"] - status["start-time"]
+                    }
         else:
             services["swiftui-upload-runner"] = {"status": "Nonexistent"}
     except ServerDisconnectedError:
         _set_error_status(request, services, "swiftui-upload-runner")
+        _set_error_status(request, services, "vault")
     except Exception as e:
         request.app["Log"].info(f"Health failed for reason: {e}")
         _set_error_status(request, services, "swiftui-upload-runner")
+        _set_error_status(request, services, "vault")
+
+
+async def get_redis(
+    services: typing.Dict[str, typing.Any],
+    request: aiohttp.web.Request,
+    performance: typing.Dict[str, typing.Any],
+) -> None:
+    """Poll Redis service."""
+    try:
+        start = time.time()
+        redis_client = await get_redis_client()
+        await redis_client.ping()
+        services["redis"] = {"status": "Ok"}
+        performance["redis"] = {"time": time.time() - start}
+        await redis_client.close()
+    except ConnectionError:
+        services["redis"] = {"status": "Down"}
+        performance["redis"] = {"time": time.time() - start}
+    except Exception as e:
+        request.app["Log"].info(f"Health failed for reason: {e}")
+        _set_error_status(request, services, "redis")
 
 
 async def handle_health_check(request: aiohttp.web.Request) -> aiohttp.web.Response:
@@ -140,6 +173,8 @@ async def handle_health_check(request: aiohttp.web.Request) -> aiohttp.web.Respo
 
     await get_upload_runner(services, request, web_client, api_params, performance)
 
+    await get_redis(services, request, performance)
+
     status["services"] = services
     status["performance"] = performance
 

swift_browser_ui/ui/server.py

@@ -4,7 +4,6 @@
 import asyncio
 import base64
 import logging
-import os
 import secrets
 import ssl
 import sys
@@ -14,12 +13,11 @@
 import aiohttp_session
 import aiohttp_session.redis_storage
 import cryptography.fernet
-import redis.asyncio as redis
 import uvloop
 from oidcrp.rp_handler import RPHandler
-from redis.asyncio.sentinel import Sentinel
 
 import swift_browser_ui.ui.middlewares
+from swift_browser_ui.ui._convenience import get_redis_client
 from swift_browser_ui.ui.api import (
     add_project_container_acl,
     close_upload_session,
@@ -113,33 +111,7 @@ async def on_prepare(
     app.on_response_prepare.append(on_prepare)
 
     # Initialize aiohttp_session
-    sentinel_url = str(os.environ.get("SWIFT_UI_REDIS_SENTINEL_HOST", ""))
-    # we make this str to make it easier to check if exists
-    sentinel_port = str(os.environ.get("SWIFT_UI_REDIS_SENTINEL_PORT", ""))
-    sentinel_master = os.environ.get("SWIFT_UI_REDIS_SENTINEL_MASTER", "mymaster")
-
-    redis_user = str(os.environ.get("SWIFT_UI_REDIS_USER", ""))
-    redis_password = str(os.environ.get("SWIFT_UI_REDIS_PASSWORD", ""))
-
-    redis_client: redis.Redis[typing.Any]
-    if sentinel_url and sentinel_port:
-        # we forward the auth to redis so no need for auth on sentinel
-        sentinel = Sentinel([(str(sentinel_url), int(sentinel_port))])
-
-        redis_client = sentinel.master_for(
-            service_name=sentinel_master,
-            redis_class=redis.Redis,
-            password=redis_password,
-            username=redis_user,
-        )
-    else:
-        redis_port = str(os.environ.get("SWIFT_UI_REDIS_PORT", ""))
-        redis_host = str(os.environ.get("SWIFT_UI_REDIS_HOST", "localhost"))
-
-        redis_creds = ""
-        if redis_user and redis_password:
-            redis_creds = f"{redis_user}:{redis_password}@"
-        redis_client = redis.from_url(f"redis://{redis_creds}{redis_host}:{redis_port}")
+    redis_client = await get_redis_client()
     storage = aiohttp_session.redis_storage.RedisStorage(
         redis_client,
         cookie_name="SWIFT_UI_SESSION",

swift_browser_ui/upload/api.py

@@ -6,6 +6,7 @@
 import json
 import logging
 import os
+import time
 import typing
 
 import aiohttp.web
@@ -292,14 +293,22 @@ async def handle_get_container(
     return resp
 
 
-async def handle_health_check(_: aiohttp.web.Request) -> aiohttp.web.Response:
-    """Answer a service health check."""
-    # Case degraded
+async def handle_health_check(request: aiohttp.web.Request) -> aiohttp.web.Response:
+    """Answer a service health check for the upload runner and vault client."""
+    start_time = time.time()
+    vault_client: VaultClient = request.app[VAULT_CLIENT]
+    try:
+        vault_status = await vault_client.get_sys_health()
+    except Exception:
+        vault_status = "Error"
+    end_time = time.time()
 
-    # Case nominal
     return aiohttp.web.json_response(
         {
-            "status": "Ok",
+            "upload-runner": {"status": "Ok"},
+            "vault-instance": {"status": vault_status},
+            "start-time": start_time,
+            "end-time": end_time,
         }
     )
 

tests/ui_unit/test_health.py

@@ -1,7 +1,8 @@
 """Module for testing ``swift_browser_ui.ui.health``."""
 
-
+import redis
 import unittest
+import unittest.mock
 
 import tests.common.mockups
 import swift_browser_ui.ui.health
@@ -24,6 +25,10 @@ def setUp(self):
         }
         self.setd = unittest.mock.patch("swift_browser_ui.ui.health.setd", self.mock_setd)
 
+        self.mock_redis = unittest.mock.AsyncMock()
+        self.mock_redis.ping.return_value = True
+        self.mock_redis_client = unittest.mock.AsyncMock(return_value=self.mock_redis)
+
     async def test_get_x_account_sharing(self):
         """Test getting x account sharing."""
         with self.setd:
@@ -86,6 +91,10 @@ async def test_get_swift_sharing(self):
 
     async def test_get_upload_runner(self):
         """Test getting upload runner."""
+        self.mock_client_json["upload-runner"] = {"status": "Ok"}
+        self.mock_client_json["vault-instance"] = {"status": "Ok"}
+        self.mock_client_json["start-time"] = 0.1
+        self.mock_client_json["end-time"] = 0.2
         with self.setd:
             await swift_browser_ui.ui.health.get_upload_runner(
                 self.mock_services,
@@ -94,10 +103,13 @@ async def test_get_upload_runner(self):
                 self.mock_api_params,
                 self.mock_performance,
             )
+
         self.mock_client.get.assert_called_once()
         self.assertEqual(self.mock_services["swiftui-upload-runner"], {"status": "Ok"})
+        self.assertEqual(self.mock_services["vault"], {"status": "Ok"})
         self.assertIn("time", self.mock_performance["swiftui-upload-runner"])
-        first_time = self.mock_performance["swiftui-upload-runner"]["time"]
+        first_time_upload = self.mock_performance["swiftui-upload-runner"]["time"]
+        first_time_vault = self.mock_performance["vault"]["time"]
 
         self.mock_client_response.status = 503
         with self.setd:
@@ -109,9 +121,40 @@ async def test_get_upload_runner(self):
                 self.mock_performance,
             )
         self.assertEqual(self.mock_services["swiftui-upload-runner"], {"status": "Down"})
+        self.assertEqual(self.mock_services["vault"], {"status": "Down"})
         self.assertNotEqual(
-            first_time, self.mock_performance["swiftui-upload-runner"]["time"]
+            first_time_upload, self.mock_performance["swiftui-upload-runner"]["time"]
         )
+        self.assertNotEqual(first_time_vault, self.mock_performance["vault"]["time"])
+
+    async def test_get_redis(self):
+        """Test getting redis service."""
+        with self.setd:
+            with unittest.mock.patch(
+                "swift_browser_ui.ui.health.get_redis_client", self.mock_redis_client
+            ):
+                await swift_browser_ui.ui.health.get_redis(
+                    self.mock_services,
+                    self.mock_request,
+                    self.mock_performance,
+                )
+        self.assertEqual(self.mock_services["redis"], {"status": "Ok"})
+        self.assertIn("time", self.mock_performance["redis"])
+        first_time = self.mock_performance["redis"]["time"]
+
+        self.mock_redis.ping.side_effect = redis.ConnectionError
+        self.mock_redis_client = unittest.mock.AsyncMock(return_value=self.mock_redis)
+        with self.setd:
+            with unittest.mock.patch(
+                "swift_browser_ui.ui.health.get_redis_client", self.mock_redis_client
+            ):
+                await swift_browser_ui.ui.health.get_redis(
+                    self.mock_services,
+                    self.mock_request,
+                    self.mock_performance,
+                )
+        self.assertEqual(self.mock_services["redis"], {"status": "Down"})
+        self.assertNotEqual(first_time, self.mock_performance["redis"]["time"])
 
     async def test_some_exception(self):
         """Test that an exception happens when getting one of the service statuses."""
@@ -143,7 +186,17 @@ async def test_nonexistant_service(self):
 
     async def test_handle_health_check(self):
         """Test handling health check."""
-        with self.setd, self.p_json_resp:
+        self.mock_client_response.json.return_value = {
+            "status": "Ok",
+            "upload-runner": {"status": "Ok"},
+            "vault-instance": {"status": "Ok"},
+            "start-time": 0.1,
+            "end-time": 0.2,
+        }
+        redis_mock = unittest.mock.patch(
+            "swift_browser_ui.ui.health.get_redis_client", new=self.mock_redis_client
+        )
+        with self.setd, self.p_json_resp, redis_mock:
             await swift_browser_ui.ui.health.handle_health_check(self.mock_request)
         self.aiohttp_json_response_mock.assert_called()
         status = self.aiohttp_json_response_mock.call_args_list[0][0][0]["status"]

tests/upload/test_api.py

@@ -67,6 +67,12 @@ def setUp(self) -> None:
         )
         self.mock_init_download = unittest.mock.Mock(return_value=self.mock_download)
 
+        self.mock_get_sys_health = unittest.mock.AsyncMock(return_value=True)
+        self.mock_vault_client = types.SimpleNamespace(
+            **{"get_sys_health": self.mock_get_sys_health}
+        )
+        self.mock_init_vault = unittest.mock.Mock(return_value=self.mock_vault_client)
+
     async def test_handle_get_object(self):
         """Test swift_browser_ui.upload.api.handle_get_object."""
         self.mock_request.match_info["project"] = "test-project"
@@ -277,7 +283,13 @@ async def test_handle_get_container(self):
 
     async def test_handle_health_check(self):
         """Test swift_browser_ui.upload.api.handle_health_check."""
-        resp = await swift_browser_ui.upload.api.handle_health_check(
-            tests.common.mockups.Mock_Request()
+        patch_init_vault = unittest.mock.patch(
+            "swift_browser_ui.upload.api.VaultClient",
+            self.mock_init_vault,
         )
+        self.mock_request.app["vault_client"] = patch_init_vault
+        with patch_init_vault:
+            resp = await swift_browser_ui.upload.api.handle_health_check(
+                self.mock_request
+            )
         self.assertIsInstance(resp, aiohttp.web.Response)