watchdog: improve dlt watchdog

CMakeLists.txt

@@ -47,6 +47,8 @@ endif()
 option(BUILD_SHARED_LIBS      "Set to OFF to build static libraries"                                             ON)
 option(WITH_SYSTEMD           "Set to ON to create unit files and systemd check on dlt-daemon startup"           OFF)
 option(WITH_SYSTEMD_WATCHDOG  "Set to ON to use the systemd watchdog in dlt-daemon"                              OFF)
+option(WITH_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX "Set to ON to trigger systemd watchdog of dlt-daemon only when a message was received since the last trigger" OFF)
+option(WITH_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_DLT_SYSTEM "Set to ON to trigger systemd watchdog of dlt-system only when a message was received since the last trigger" OFF)
 option(WITH_SYSTEMD_JOURNAL   "Set to ON to use the systemd journal in dlt-system"                               OFF)
 option(WITH_SYSTEMD_SOCKET_ACTIVATION "Set to ON to use systemd socket activation"                               OFF)
 option(WITH_DOC               "Set to ON to build documentation target"                                          OFF)
@@ -56,7 +58,7 @@ option(WITH_TESTSCRIPTS       "Set to ON to run CMakeLists.txt in testscripts"
 option(WITH_GPROF             "Set -pg to compile flags"                                                         OFF)
 option(WITH_DLTTEST           "Set to ON to build with modifications to test User-Daemon communication with corrupt messages" OFF)
 option(WITH_DLT_SHM_ENABLE    "EXPERIMENTAL! Set to ON to use shared memory as IPC. EXPERIMENTAL!"               OFF)
-option(WITH_DLT_ADAPTOR       "Set to ON to build src/adaptor binaries"                                          OFF) 
+option(WITH_DLT_ADAPTOR       "Set to ON to build src/adaptor binaries"                                          OFF)
 option(WITH_DLT_ADAPTOR_STDIN "Set to ON to build src/adaptor/stdin binaries"                                    OFF)
 option(WITH_DLT_ADAPTOR_UDP   "Set to ON to build src/adaptor/udp binaries"                                      OFF)
 option(WITH_DLT_CONSOLE       "Set to ON to build src/console binaries"                                          ON)
@@ -234,7 +236,13 @@ else()
     message(STATUS "Network trace interface disabled")
 endif()
 
-if(WITH_SYSTEMD OR WITH_SYSTEMD_WATCHDOG OR WITH_SYSTEMD_JOURNAL OR WITH_SYSTEMD_SOCKET_ACTIVATION)
+if(WITH_SYSTEMD
+    OR WITH_SYSTEMD_WATCHDOG
+    OR WITH_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX
+    OR WITH_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_DLT_SYSTEM
+    OR WITH_SYSTEMD_JOURNAL
+    OR WITH_SYSTEMD_SOCKET_ACTIVATION)
+
     find_package(PkgConfig REQUIRED)
     execute_process(COMMAND pkg-config --modversion systemd OUTPUT_VARIABLE SYSTEMD_VERSION)
     string(REPLACE "\n" "" SYSTEMD_VERSION ${SYSTEMD_VERSION})
@@ -251,6 +259,14 @@ if(WITH_SYSTEMD OR WITH_SYSTEMD_WATCHDOG OR WITH_SYSTEMD_JOURNAL OR WITH_SYSTEMD
         add_definitions(-DDLT_SYSTEMD_JOURNAL_ENABLE)
     endif()
 
+    if (WITH_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX)
+        add_definitions(-DDLT_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_ENABLE)
+    endif()
+
+    if (WITH_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_DLT_SYSTEM)
+        add_definitions(-DDLT_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_ENABLE_DLT_SYTSTEM)
+    endif()
+
     if (WITH_SYSTEMD_SOCKET_ACTIVATION)
         if(NOT DLT_IPC STREQUAL "UNIX_SOCKET")
             message(FATAL_ERROR "WITH_SYSTEMD_SOCKET_ACTIVATION is only supported for UNIX_SOCKET")
@@ -346,6 +362,8 @@ message(STATUS "TARGET_CPU_NAME = ${TARGET_CPU_NAME}")
 if(WITH_SYSTEMD OR WITH_SYSTEMD_WATCHDOG OR WITH_SYSTEMD_JOURNAL)
     message(STATUS "SYSTEMD_VERSION = ${SYSTEMD_VERSION}")
     message(STATUS "SYSTEMD_UNITDIR = ${SYSTEMD_UNITDIR}")
+    message(STATUS "WITH_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX = ${WITH_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX}" )
+    message(STATUS "WITH_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_DLT_SYSTEM = ${WITH_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_DLT_SYSTEM}" )
 endif()
 message(STATUS "CMAKE_INSTALL_PREFIX = ${CMAKE_INSTALL_PREFIX}")
 message(STATUS "CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE}")

doc/dlt_build_options.md

@@ -48,6 +48,8 @@ WITH\_EXTENDED\_FILTERING         | OFF            | Set to OFF to build without
  :--- | :--- | :---
 WITH\_SYSTEMD                     | OFF            | Set to ON to run CMakeLists.txt in systemd
 WITH\_SYSTEMD\_WATCHDOG           | OFF            | Set to ON to use the systemd watchdog in dlt-daemon
+WITH\_SYSTEMD\_WATCHDOG\_ENFORCE\_MSG\_RX | OFF    | Set to ON to notify the watchdog only if new messages where received in dlt-daemon since last notify
+WITH\_SYSTEMD\_WATCHDOG\_ENFORCE\_MSG\_RX\_DLT\_SYSTEM | OFF    | Set to ON to notify the watchdog only if new messages where received in dlt-system since last notify
 WITH\_SYSTEMD\_JOURNAL            | OFF            | Set to ON to use the systemd journal in dlt-system
 WITH\_DLT\_DBUS                   | OFF            | Set to ON to build src/dbus binaries
 

src/daemon/dlt-daemon.c

@@ -3196,6 +3196,9 @@ int dlt_daemon_process_user_message_log(DltDaemon *daemon,
         return DLT_DAEMON_ERROR_UNKNOWN;
     }
 
+#ifdef DLT_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_ENABLE
+    daemon->received_message_since_last_watchdog_interval = 1;
+#endif
 #ifdef DLT_SHM_ENABLE
 
     /** In case of SHM, the header still received via fifo/unix_socket receiver,
@@ -3451,7 +3454,7 @@ int dlt_daemon_send_ringbuffer_to_client(DltDaemon *daemon, DltDaemonLocal *daem
 #ifdef DLT_SYSTEMD_WATCHDOG_ENABLE
 
     if (sd_notify(0, "WATCHDOG=1") < 0)
-        dlt_log(LOG_WARNING, "Could not reset systemd watchdog\n");
+        dlt_vlog(LOG_WARNING, "Could not reset systemd watchdog: %s\n", strerror(errno));
 
     curr_time = dlt_uptime();
 #endif

src/daemon/dlt_daemon_client.c

@@ -2372,6 +2372,14 @@ int dlt_daemon_process_systemd_timer(DltDaemon *daemon,
          * let's go on sending notification */
     }
 
+#ifdef DLT_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_ENABLE
+    if (!daemon->received_message_since_last_watchdog_interval) {
+      dlt_log(LOG_WARNING, "No new messages received since last watchdog timer run\n");
+      return 0;
+    }
+    daemon->received_message_since_last_watchdog_interval = 0;
+#endif
+
     if (sd_notify(0, "WATCHDOG=1") < 0)
         dlt_log(LOG_CRIT, "Could not reset systemd watchdog\n");
 

src/daemon/dlt_daemon_common.c

@@ -244,6 +244,9 @@ int dlt_daemon_init(DltDaemon *daemon,
         return -1;
 
     daemon->storage_handle = NULL;
+#ifdef DLT_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_ENABLE
+    daemon->received_message_since_last_watchdog_interval = 0;
+#endif
     return 0;
 }
 

src/daemon/dlt_daemon_common.h

@@ -191,6 +191,9 @@ typedef struct
     DltDaemonState state;   /**< the current logging state of dlt daemon. */
     DltLogStorage *storage_handle;
     int maintain_logstorage_loglevel;     /* Permission to maintain the logstorage loglevel*/
+#ifdef DLT_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_ENABLE
+    int received_message_since_last_watchdog_interval;
+#endif
 } DltDaemon;
 
 /**

src/system/dlt-system-journal.c

@@ -229,6 +229,10 @@ void get_journal_msg(sd_journal *j, DltSystemConfiguration *config)
             return;
         }
 
+        #if defined(DLT_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_ENABLE_DLT_SYSTEM) && defined(DLT_SYSTEMD_JOURNAL_ENABLE)
+        config->Journal.MessageReceived = 1;
+        #endif
+
         /* get all data from current journal entry */
         dlt_system_journal_get_timestamp(j, &timestamp);
 
@@ -390,4 +394,4 @@ void journal_fd_handler(sd_journal *j, DltSystemConfiguration *config)
     get_journal_msg(j, config);
 }
 
-#endif /* DLT_SYSTEMD_JOURNAL_ENABLE */
+#endif /* DLT_SYSTEMD_JOURNAL_ENABLE */

src/system/dlt-system-process-handling.c

@@ -300,7 +300,11 @@ void start_dlt_system_processes(DltSystemConfiguration *config)
                 #endif
                 #if defined(DLT_SYSTEMD_WATCHDOG_ENABLE)
                 else if (fdType[i] == fdType_watchdog) {
+                    #if defined(DLT_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_ENABLE_DLT_SYSTEM) && defined(DLT_SYSTEMD_JOURNAL_ENABLE)
+                    watchdog_fd_handler(pollfd[i].fd, &config->Journal.MessageReceived);
+                    #else
                     watchdog_fd_handler(pollfd[i].fd);
+                    #endif
                 }
                 #endif
             }
@@ -330,4 +334,4 @@ void dlt_system_signal_handler(int sig)
                 DLT_STRING("dlt-system-process-handling, unknown signal!"));
         break;
     }
-}
+}

src/system/dlt-system-watchdog.c

@@ -107,18 +107,29 @@ int register_watchdog_fd(struct pollfd *pollfd, int fdcnt)
     return 0;
 }
 
+#if defined(DLT_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_ENABLE_DLT_SYSTEM) && defined(DLT_SYSTEMD_JOURNAL_ENABLE)
+void watchdog_fd_handler(int fd, int* received_message_since_last_watchdog_interval)
+#else
 void watchdog_fd_handler(int fd)
+#endif
 {
     uint64_t timersElapsed = 0ULL;
     int r = read(fd, &timersElapsed, 8U);    // only needed to reset fd event
     if(r < 0)
         DLT_LOG(watchdogContext, DLT_LOG_ERROR, DLT_STRING("Could not reset systemd watchdog. Exit with: "), 
             DLT_STRING(strerror(r)));
 
+    #ifdef DLT_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_ENABLE_DLT_SYSTEM
+    if (!*received_message_since_last_watchdog_interval) {
+      dlt_log(LOG_WARNING, "No new messages received since last watchdog timer run\n");
+      return;
+    }
+    *received_message_since_last_watchdog_interval = 0;
+    #endif
+
     if (sd_notify(0, "WATCHDOG=1") < 0)
         DLT_LOG(watchdogContext, DLT_LOG_ERROR, DLT_STRING("Could not reset systemd watchdog\n"));
 
     DLT_LOG(watchdogContext, DLT_LOG_DEBUG, DLT_STRING("systemd watchdog waited periodic\n"));
 }
 #endif
-

src/system/dlt-system.h

@@ -127,6 +127,9 @@ typedef struct {
     int Follow;
     int MapLogLevels;
     int UseOriginalTimestamp;
+#ifdef DLT_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_ENABLE_DLT_SYSTEM
+    int MessageReceived;
+#endif
 } JournalOptions;
 
 typedef struct {
@@ -209,7 +212,11 @@ int register_syslog_fd(struct pollfd *pollfd, int i, DltSystemConfiguration *con
 void logfile_fd_handler(void *v_conf);
 void logprocess_fd_handler(void *v_conf);
 void filetransfer_fd_handler(DltSystemConfiguration *config);
+#if defined(DLT_SYSTEMD_WATCHDOG_ENFORCE_MSG_RX_ENABLE_DLT_SYSTEM) && defined(DLT_SYSTEMD_JOURNAL_ENABLE)
+void watchdog_fd_handler(int fd, int* received_message_since_last_watchdog_interval);
+#else
 void watchdog_fd_handler(int fd);
+#endif
 void journal_fd_handler(sd_journal *j, DltSystemConfiguration *config);
 void syslog_fd_handler(int syslogSock);