Re-connect to Zino server on lost connection

pyproject.toml

@@ -20,7 +20,7 @@ dependencies = [
     "flask_assets",
     "flask-login",
     "jinja2",
-    "zinolib>=1.1.0",
+    "zinolib>=1.1.1",
     "werkzeug<3",  # needed by flask-login
     "pydantic",
 ]

requirements-frozen.txt

@@ -49,5 +49,5 @@ werkzeug==2.3.8
     #   flask
     #   flask-login
     #   howitz (pyproject.toml)
-zinolib==1.1.0
+zinolib==1.1.1
     # via howitz (pyproject.toml)

src/howitz/__init__.py

@@ -12,10 +12,10 @@
 from howitz.config.utils import load_config
 from howitz.config.zino1 import make_zino1_config
 from howitz.config.howitz import make_howitz_config
-from howitz.error_handlers import handle_generic_exception, handle_generic_http_exception, handle_400, handle_404, handle_403
+from howitz.error_handlers import handle_generic_exception, handle_generic_http_exception, handle_400, handle_404, handle_403, handle_lost_connection
 from howitz.users.db import UserDB
 from howitz.users.commands import user_cli
-from zinolib.controllers.zino1 import Zino1EventManager
+from zinolib.controllers.zino1 import Zino1EventManager, LostConnectionError, NotConnectedError
 
 __all__ = ["create_app"]
 
@@ -28,7 +28,10 @@ def create_app(test_config=None):
     app.register_error_handler(HTTPException, handle_generic_http_exception)
     app.register_error_handler(BadRequest, handle_400)
     app.register_error_handler(NotFound, handle_404)
-    app.register_error_handler(Forbidden, handle_403),
+    app.register_error_handler(Forbidden, handle_403)
+    app.register_error_handler(LostConnectionError, handle_lost_connection)
+    app.register_error_handler(BrokenPipeError, handle_lost_connection)
+    app.register_error_handler(NotConnectedError, handle_lost_connection)
 
     # load config
     app = load_config(app, test_config)

src/howitz/endpoints.py

@@ -20,7 +20,7 @@
 from zinolib.controllers.zino1 import Zino1EventManager, RetryError, EventClosedError, UpdateHandler
 from zinolib.event_types import Event, AdmState, PortState, BFDState, ReachabilityState, LogEntry, HistoryEntry
 from zinolib.compat import StrEnum
-from zinolib.ritz import NotConnectedError, AuthenticationError
+from zinolib.ritz import AuthenticationError
 
 from howitz.users.utils import authenticate_user
 
@@ -45,28 +45,16 @@ def auth_handler(username, password):
         user = authenticate_user(current_app.database, username, password)
         current_app.logger.debug('User %s', user)
 
-        if not current_app.event_manager.is_connected:
-            current_app.event_manager = Zino1EventManager.configure(current_app.zino_config)
-            current_app.event_manager.connect()
-            current_app.logger.info('Connected to Zino %s', current_app.event_manager.is_connected)
-
-        if not current_app.event_manager.is_authenticated:
-            current_app.event_manager.authenticate(username=user.username, password=user.token)
-            current_app.logger.info('Authenticated in Zino %s', current_app.event_manager.is_authenticated)
+        connect_to_zino(user.username, user.token)
 
         if current_app.event_manager.is_authenticated:  # is zino authenticated
-            current_app.updater = UpdateHandler(current_app.event_manager, autoremove=current_app.zino_config.autoremove)
-            current_app.updater.connect()
-            current_app.logger.debug('UpdateHandler %s', current_app.updater)
-
             current_app.logger.debug('User is Zino authenticated %s', current_app.event_manager.is_authenticated)
             current_app.logger.debug('HOWITZ CONFIG %s', current_app.howitz_config)
             login_user(user, remember=True)
             flash('Logged in successfully.')
             session["selected_events"] = []
             session["expanded_events"] = {}
             session["errors"] = {}
-            session["not_connected_counter"] = 0
             session["event_ids"] = []
             return user
 
@@ -83,11 +71,26 @@ def logout_handler():
         session.pop('expanded_events', {})
         session.pop('selected_events', [])
         session.pop('errors', {})
-        session.pop('not_connected_counter', 0)
         session.pop('event_ids', [])
         current_app.logger.info("Logged out successfully.")
 
 
+def connect_to_zino(username, token):
+    if not current_app.event_manager.is_connected:
+        current_app.event_manager = Zino1EventManager.configure(current_app.zino_config)
+        current_app.event_manager.connect()
+        current_app.logger.info('Connected to Zino %s', current_app.event_manager.is_connected)
+
+    if not current_app.event_manager.is_authenticated:
+        current_app.event_manager.authenticate(username=username, password=token)
+        current_app.logger.info('Authenticated in Zino %s', current_app.event_manager.is_authenticated)
+
+    if current_app.event_manager.is_authenticated:  # is zino authenticated
+        current_app.updater = UpdateHandler(current_app.event_manager, autoremove=current_app.zino_config.autoremove)
+        current_app.updater.connect()
+        current_app.logger.debug('UpdateHandler %s', current_app.updater)
+
+
 def clear_ui_state():
     session["selected_events"] = []
     session["expanded_events"] = {}
@@ -107,17 +110,6 @@ def get_current_events():
         except RetryError as retryErr:  # Intermittent error in Zino
             current_app.logger.exception('RetryError when fetching current events after retry, %s', retryErr)
             raise
-    except NotConnectedError as notConnErr:
-        if session["not_connected_counter"] > 1:  # This error is not intermittent - increase counter and handle
-            current_app.logger.exception('Recurrent NotConnectedError %s', notConnErr)
-            session["not_connected_counter"] += 1
-            raise
-        else:  # This error is intermittent - increase counter and retry
-            current_app.logger.exception('Intermittent NotConnectedError %s', notConnErr)
-            session["not_connected_counter"] += 1
-            current_app.event_manager.get_events()
-            pass
-
     events = current_app.event_manager.events
 
     events_sorted = {k: events[k] for k in sorted(events,

src/howitz/error_handlers.py

@@ -1,8 +1,10 @@
 import uuid
 
 from flask import render_template, session, current_app, make_response, request
+from flask_login import current_user
 from werkzeug.exceptions import HTTPException
 
+from howitz.endpoints import connect_to_zino
 from howitz.utils import serialize_exception
 
 
@@ -75,3 +77,36 @@ def handle_403(e):
     response.headers['HX-Trigger'] = 'htmx:responseError'
 
     return response, 403
+
+
+def handle_lost_connection(e):
+    if isinstance(e, BrokenPipeError):
+        current_app.logger.exception("Lost connection to Zino server: %s", e)
+    else:
+        current_app.logger.error("Lost connection to Zino server: %s", e.args[0])
+
+    if current_user.is_authenticated:  # Re-connect to Zino with existing credentials and inform user that there was an error via alert pop-up
+        if current_app.event_manager.is_connected:
+            current_app.event_manager.disconnect()
+
+        connect_to_zino(current_user.username, current_user.token)
+
+        alert_random_id = str(uuid.uuid4())
+        try:
+            short_err_msg = e.args[0]
+        except IndexError:
+            short_err_msg = 'Temporarily lost connection to Zino server'
+
+        if not "errors" in session:
+            session["errors"] = dict()
+        session["errors"][str(alert_random_id)] = serialize_exception(e)
+        session.modified = True
+
+        response = make_response(render_template('/components/popups/alerts/error/error-alert.html',
+                                                 alert_id=alert_random_id, short_err_msg=short_err_msg))
+        response.headers['HX-Reswap'] = 'beforeend'
+        return response, 503
+    else:  # Redirect to /login for complete re-authentication
+        res = make_response()
+        res.headers['HX-Redirect'] = '/login'
+        return res