watchdog: improve dlt watchdog

* watchdog can be configured to notify systemd only if messages are received * this can activated via -DWITH_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX=ON Signed-off-by: Alexander Mohr <[email protected]>
COVESA · May 17, 2023 · d7ac962 · d7ac962
1 parent 0c1c75e
commit d7ac962
Show file tree

Hide file tree

Showing 10 changed files with 67 additions and 6 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -47,6 +47,8 @@ endif()
 option(BUILD_SHARED_LIBS      "Set to OFF to build static libraries"                                             ON)
 option(WITH_SYSTEMD           "Set to ON to create unit files and systemd check on dlt-daemon startup"           OFF)
 option(WITH_SYSTEMD_WATCHDOG  "Set to ON to use the systemd watchdog in dlt-daemon"                              OFF)
+option(WITH_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX "Set to ON to trigger systemd watchdog of dlt-daemon only when a message was received since the last trigger" OFF)
+option(WITH_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_DLT_SYSTEM "Set to ON to trigger systemd watchdog of dlt-system only when a message was received since the last trigger" OFF)
 option(WITH_SYSTEMD_JOURNAL   "Set to ON to use the systemd journal in dlt-system"                               OFF)
 option(WITH_SYSTEMD_SOCKET_ACTIVATION "Set to ON to use systemd socket activation"                               OFF)
 option(WITH_DOC               "Set to ON to build documentation target"                                          OFF)
@@ -56,7 +58,7 @@ option(WITH_TESTSCRIPTS       "Set to ON to run CMakeLists.txt in testscripts"
 option(WITH_GPROF             "Set -pg to compile flags"                                                         OFF)
 option(WITH_DLTTEST           "Set to ON to build with modifications to test User-Daemon communication with corrupt messages" OFF)
 option(WITH_DLT_SHM_ENABLE    "EXPERIMENTAL! Set to ON to use shared memory as IPC. EXPERIMENTAL!"               OFF)
-option(WITH_DLT_ADAPTOR       "Set to ON to build src/adaptor binaries"                                          OFF) 
+option(WITH_DLT_ADAPTOR       "Set to ON to build src/adaptor binaries"                                          OFF)
 option(WITH_DLT_ADAPTOR_STDIN "Set to ON to build src/adaptor/stdin binaries"                                    OFF)
 option(WITH_DLT_ADAPTOR_UDP   "Set to ON to build src/adaptor/udp binaries"                                      OFF)
 option(WITH_DLT_CONSOLE       "Set to ON to build src/console binaries"                                          ON)
@@ -234,7 +236,13 @@ else()
     message(STATUS "Network trace interface disabled")
 endif()
 
-if(WITH_SYSTEMD OR WITH_SYSTEMD_WATCHDOG OR WITH_SYSTEMD_JOURNAL OR WITH_SYSTEMD_SOCKET_ACTIVATION)
+if(WITH_SYSTEMD
+    OR WITH_SYSTEMD_WATCHDOG
+    OR WITH_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX
+    OR WITH_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_DLT_SYSTEM
+    OR WITH_SYSTEMD_JOURNAL
+    OR WITH_SYSTEMD_SOCKET_ACTIVATION)
+
     find_package(PkgConfig REQUIRED)
     execute_process(COMMAND pkg-config --modversion systemd OUTPUT_VARIABLE SYSTEMD_VERSION)
     string(REPLACE "\n" "" SYSTEMD_VERSION ${SYSTEMD_VERSION})
@@ -251,6 +259,14 @@ if(WITH_SYSTEMD OR WITH_SYSTEMD_WATCHDOG OR WITH_SYSTEMD_JOURNAL OR WITH_SYSTEMD
         add_definitions(-DDLT_SYSTEMD_JOURNAL_ENABLE)
     endif()
 
+    if (WITH_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX)
+        add_definitions(-DDLT_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_ENABLE)
+    endif()
+
+    if (WITH_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_DLT_SYSTEM)
+        add_definitions(-DDLT_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_ENABLE_DLT_SYTSTEM)
+    endif()
+
     if (WITH_SYSTEMD_SOCKET_ACTIVATION)
         if(NOT DLT_IPC STREQUAL "UNIX_SOCKET")
             message(FATAL_ERROR "WITH_SYSTEMD_SOCKET_ACTIVATION is only supported for UNIX_SOCKET")

diff --git a/doc/dlt_build_options.md b/doc/dlt_build_options.md
@@ -48,6 +48,8 @@ WITH\_EXTENDED\_FILTERING         | OFF            | Set to OFF to build without
  :--- | :--- | :---
 WITH\_SYSTEMD                     | OFF            | Set to ON to run CMakeLists.txt in systemd
 WITH\_SYSTEMD\_WATCHDOG           | OFF            | Set to ON to use the systemd watchdog in dlt-daemon
+WITH\_SYSTEMD\_WATCHDOG\_ENFORCE\_MSG\_RX | OFF    | Set to ON to notify the watchdog only if new messages where received in dlt-daemon since last notify
+WITH\_SYSTEMD\_WATCHDOG\_ENFORCE\_MSG\_RX\_DLT\_SYSTEM | OFF    | Set to ON to notify the watchdog only if new messages where received in dlt-system since last notify
 WITH\_SYSTEMD\_JOURNAL            | OFF            | Set to ON to use the systemd journal in dlt-system
 WITH\_DLT\_DBUS                   | OFF            | Set to ON to build src/dbus binaries
 

diff --git a/src/daemon/dlt-daemon.c b/src/daemon/dlt-daemon.c
@@ -3196,6 +3196,9 @@ int dlt_daemon_process_user_message_log(DltDaemon *daemon,
         return DLT_DAEMON_ERROR_UNKNOWN;
     }
 
+#ifdef DLT_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_ENABLE
+    daemon->received_message_since_last_watchdog_interval = 1;
+#endif
 #ifdef DLT_SHM_ENABLE
 
     /** In case of SHM, the header still received via fifo/unix_socket receiver,
@@ -3451,7 +3454,7 @@ int dlt_daemon_send_ringbuffer_to_client(DltDaemon *daemon, DltDaemonLocal *daem
 #ifdef DLT_SYSTEMD_WATCHDOG_ENABLE
 
     if (sd_notify(0, "WATCHDOG=1") < 0)
-        dlt_log(LOG_WARNING, "Could not reset systemd watchdog\n");
+        dlt_vlog(LOG_WARNING, "Could not reset systemd watchdog: %s\n", strerror(errno));
 
     curr_time = dlt_uptime();
 #endif

diff --git a/src/daemon/dlt_daemon_client.c b/src/daemon/dlt_daemon_client.c
@@ -2372,6 +2372,14 @@ int dlt_daemon_process_systemd_timer(DltDaemon *daemon,
          * let's go on sending notification */
     }
 
+#ifdef DLT_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_ENABLE
+    if (!daemon->received_message_since_last_watchdog_interval) {
+      dlt_log(LOG_WARNING, "No new messages received since last watchdog timer run\n");
+      return 0;
+    }
+    daemon->received_message_since_last_watchdog_interval = 0;
+#endif
+
     if (sd_notify(0, "WATCHDOG=1") < 0)
         dlt_log(LOG_CRIT, "Could not reset systemd watchdog\n");
 

diff --git a/src/daemon/dlt_daemon_common.c b/src/daemon/dlt_daemon_common.c
@@ -244,6 +244,9 @@ int dlt_daemon_init(DltDaemon *daemon,
         return -1;
 
     daemon->storage_handle = NULL;
+#ifdef DLT_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_ENABLE
+    daemon->received_message_since_last_watchdog_interval = 0;
+#endif
     return 0;
 }
 

diff --git a/src/daemon/dlt_daemon_common.h b/src/daemon/dlt_daemon_common.h
@@ -191,6 +191,9 @@ typedef struct
     DltDaemonState state;   /**< the current logging state of dlt daemon. */
     DltLogStorage *storage_handle;
     int maintain_logstorage_loglevel;     /* Permission to maintain the logstorage loglevel*/
+#ifdef DLT_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_ENABLE
+    int received_message_since_last_watchdog_interval;
+#endif
 } DltDaemon;
 
 /**

diff --git a/src/system/dlt-system-journal.c b/src/system/dlt-system-journal.c
@@ -229,6 +229,10 @@ void get_journal_msg(sd_journal *j, DltSystemConfiguration *config)
             return;
         }
 
+        #if defined(DLT_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_ENABLE_DLT_SYSTEM) && defined(DLT_SYSTEMD_JOURNAL_ENABLE)
+        config->Journal.MessageReceived = 1;
+        #endif
+
         /* get all data from current journal entry */
         dlt_system_journal_get_timestamp(j, &timestamp);
 
@@ -390,4 +394,4 @@ void journal_fd_handler(sd_journal *j, DltSystemConfiguration *config)
     get_journal_msg(j, config);
 }
 
-#endif /* DLT_SYSTEMD_JOURNAL_ENABLE */
+#endif /* DLT_SYSTEMD_JOURNAL_ENABLE */
diff --git a/src/system/dlt-system-process-handling.c b/src/system/dlt-system-process-handling.c
@@ -300,7 +300,11 @@ void start_dlt_system_processes(DltSystemConfiguration *config)
                 #endif
                 #if defined(DLT_SYSTEMD_WATCHDOG_ENABLE)
                 else if (fdType[i] == fdType_watchdog) {
+                    #if defined(DLT_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_ENABLE_DLT_SYSTEM) && defined(DLT_SYSTEMD_JOURNAL_ENABLE)
+                    watchdog_fd_handler(pollfd[i].fd, &config->Journal.MessageReceived);
+                    #else
                     watchdog_fd_handler(pollfd[i].fd);
+                    #endif
                 }
                 #endif
             }
@@ -330,4 +334,4 @@ void dlt_system_signal_handler(int sig)
                 DLT_STRING("dlt-system-process-handling, unknown signal!"));
         break;
     }
-}
+}
diff --git a/src/system/dlt-system-watchdog.c b/src/system/dlt-system-watchdog.c
@@ -107,18 +107,29 @@ int register_watchdog_fd(struct pollfd *pollfd, int fdcnt)
     return 0;
 }
 
+#if defined(DLT_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_ENABLE_DLT_SYSTEM) && defined(DLT_SYSTEMD_JOURNAL_ENABLE)
+void watchdog_fd_handler(int fd, int* received_message_since_last_watchdog_interval)
+#else
 void watchdog_fd_handler(int fd)
+#endif
 {
     uint64_t timersElapsed = 0ULL;
     int r = read(fd, &timersElapsed, 8U);    // only needed to reset fd event
     if(r < 0)
         DLT_LOG(watchdogContext, DLT_LOG_ERROR, DLT_STRING("Could not reset systemd watchdog. Exit with: "), 
             DLT_STRING(strerror(r)));
 
+    #ifdef DLT_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_ENABLE_DLT_SYSTEM
+    if (!*received_message_since_last_watchdog_interval) {
+      dlt_log(LOG_WARNING, "No new messages received since last watchdog timer run\n");
+      return;
+    }
+    *received_message_since_last_watchdog_interval = 0;
+    #endif
+
     if (sd_notify(0, "WATCHDOG=1") < 0)
         DLT_LOG(watchdogContext, DLT_LOG_ERROR, DLT_STRING("Could not reset systemd watchdog\n"));
 
     DLT_LOG(watchdogContext, DLT_LOG_DEBUG, DLT_STRING("systemd watchdog waited periodic\n"));
 }
 #endif
-
diff --git a/src/system/dlt-system.h b/src/system/dlt-system.h
@@ -127,6 +127,9 @@ typedef struct {
     int Follow;
     int MapLogLevels;
     int UseOriginalTimestamp;
+#ifdef DLT_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_ENABLE_DLT_SYSTEM
+    int MessageReceived;
+#endif
 } JournalOptions;
 
 typedef struct {
@@ -209,7 +212,11 @@ int register_syslog_fd(struct pollfd *pollfd, int i, DltSystemConfiguration *con
 void logfile_fd_handler(void *v_conf);
 void logprocess_fd_handler(void *v_conf);
 void filetransfer_fd_handler(DltSystemConfiguration *config);
+#if defined(DLT_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_ENABLE_DLT_SYSTEM) && defined(DLT_SYSTEMD_JOURNAL_ENABLE)
+void watchdog_fd_handler(int fd, int* received_message_since_last_watchdog_interval);
+#else
 void watchdog_fd_handler(int fd);
+#endif
 void journal_fd_handler(sd_journal *j, DltSystemConfiguration *config);
 void syslog_fd_handler(int syslogSock);