From bb6c5774c843dbfad5f1ba00ee76dae7720902d1 Mon Sep 17 00:00:00 2001
From: maksymbelei95 <75987222+maksymbelei95@users.noreply.github.com>
Date: Thu, 25 Nov 2021 14:20:21 +0200
Subject: [PATCH] [warm-reboot] Fix failures of warm reboot on disconnect of
 ssh session (#1529)

Starting the script in background mode and detaching the process from terminal session to prevent failures, caused by closing or sudden disconnecting of terminal session.
Redirecting output of the script to the file to prevent failures, when the script tries to write an output to file descriptor of
the nonexistent terminal session.
Adding new parameter to script to be able to explicitly run the script in foreground mode with output to the terminal.
Updating the command reference doc according to the changes.

- What I did
Resolves Azure/sonic-buildimage#7127
Fixed failures of warm reboot, when the SSH session is being disconnected.
As the script will now be executed in background mode by default, added parameter to explicitly run it as usual, in foreground mode.
Updated command reference according to the changes.

- How I did it
By restarting the script in background mode with detaching it from the terminal session.
All the output has redirected to file /var/log/warm-reboot.txt for warm-reboot case, or /var/log/fast-reboot.txt for fast-reboot, depends on REBOOT_TYPE. This will prevent crashes of the script in case, when it will try to write some data to the file descriptor of the disconnected terminal session.

- How to verify it
1. Connect to the switch with SSH;
2. Execute sudo warm-reboot -v;
3. See the current progress of warm reboot with cat /var/log/warm-reboot.txt;
4. Close SSH connection before warm reboot finish;
    Warm reboot should finish successfully, in spite of status of the SSH session.

- New command output (if the output of a command-line utility has changed)
The script will be running in background detached mode with output to the file. The related log will be shown in terminal before restarting in background mode:

admin@sonic:~$ sudo warm-reboot
Detaching the process from the terminal session. Redirecting output to /var/log/warm-reboot.txt.

All the usual logs will be written to warm-reboot.txt.
---
 doc/Command-Reference.md |  3 ++-
 scripts/fast-reboot      | 18 +++++++++++++++++-
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md
index 94db3a048c..69a3fb98b8 100644
--- a/doc/Command-Reference.md
+++ b/doc/Command-Reference.md
@@ -8944,7 +8944,7 @@ This command requires root privilege.
 
 - Usage:
   ```
-  warm-reboot [-h|-?|-v|-f|-r|-k|-x|-c <control plane assistant IP list>|-s]
+  warm-reboot [-h|-?|-v|-f|-r|-k|-x|-c <control plane assistant IP list>|-s|-D]
   ```
 
 - Parameters:
@@ -8958,6 +8958,7 @@ This command requires root privilege.
     -c    : specify control plane assistant IP list
     -s    : strict mode: do not proceed without:
             - control plane assistant IP list.
+    -D    : detached mode - closing terminal will not cause stopping reboot
   ```
 
 - Example:
diff --git a/scripts/fast-reboot b/scripts/fast-reboot
index 869a56373d..4a869a3f9f 100755
--- a/scripts/fast-reboot
+++ b/scripts/fast-reboot
@@ -25,6 +25,8 @@ LOG_SSD_HEALTH="/usr/local/bin/log_ssd_health"
 PLATFORM_FWUTIL_AU_REBOOT_HANDLE="platform_fw_au_reboot_handle"
 SSD_FW_UPDATE="ssd-fw-upgrade"
 TAG_LATEST=yes
+DETACH=no
+LOG_PATH="/var/log/${REBOOT_TYPE}.txt"
 
 # Require 100M available on the hard drive for warm reboot temp files,
 # Size is in 1K blocks:
@@ -73,13 +75,14 @@ function showHelpAndExit()
     echo "    -s    : strict mode: do not proceed without:"
     echo "            - control plane assistant IP list."
     echo "    -t    : Don't tag the current kube images as latest"
+    echo "    -D    : detached mode - closing terminal will not cause stopping reboot"
 
     exit "${EXIT_SUCCESS}"
 }
 
 function parseOptions()
 {
-    while getopts "vfidh?rkxc:s" opt; do
+    while getopts "vfidh?rkxc:sD" opt; do #TODO "t" is missing
         case ${opt} in
             h|\? )
                 showHelpAndExit
@@ -114,6 +117,9 @@ function parseOptions()
             t )
                 TAG_LATEST=no
                 ;;
+            D )
+                DETACH=yes
+                ;;
         esac
     done
 }
@@ -487,8 +493,18 @@ then
     exit "${EXIT_FAILURE}"
 fi
 
+
+# re-run the script in background mode with detaching from the terminal session
+if [[ x"${DETACH}" == x"yes" && x"${ALREADY_DETACHED}" == x"" ]]; then
+   echo "Detaching the process from the terminal session. Redirecting output to ${LOG_PATH}."
+   ALREADY_DETACHED=yes $0 "$@" &>$LOG_PATH & disown %%
+   exit $?
+fi
+
+
 check_conflict_boot_in_fw_update
 
+
 sonic_asic_type=$(sonic-cfggen -y /etc/sonic/sonic_version.yml -v asic_type)
 
 # Check reboot type supported