integrated reboot-cause history into reboot tests in platfrom_tests/t…

…est_reboot.py (sonic-net#4270)

This is to add test case in sonic-mgmt/tests/platform_tests/test_reboot.py to cover new feature reboot-cause history introduced in sonic-net/SONiC#669. Example:

show reboot-cause history
Name                 Cause          Time                             User    Comment
-------------------  -------------  -------------------------------  ------  ---------
2021_09_09_14_15_13  Power Loss ()  N/A                              N/A     N/A
2021_09_09_14_06_17  reboot         Thu 09 Sep 2021 02:05:17 PM UTC  admin   N/A
2021_09_09_13_59_11  Watchdog ()    N/A                              N/A     N/A
2021_09_09_13_52_13  Power Loss ()  N/A                              N/A     N/A
2021_09_09_13_45_18  warm-reboot    Thu 09 Sep 2021 01:44:14 PM UTC  admin   N/A
2021_09_09_13_37_58  fast-reboot    Thu 09 Sep 2021 01:37:09 PM UTC  admin   N/A
2021_09_09_13_30_52  soft-reboot    Thu 09 Sep 2021 01:30:24 PM UTC  admin   N/A
2021_09_09_13_24_17  reboot         Thu 09 Sep 2021 01:23:17 PM UTC  admin   N/A

What is the motivation for this PR?
In order to verify reboot-cause history function, add new checker for "show reboot-cause history" into reboot tests

How did you do it?
Add one new function of check_reboot_cause_history in sonic-mgmt/tests/platform_tests/test_reboot.py.
In test_reboot.py, there are 8 different reboot tests. In order to save test execution time, we leverage the reboot-cause history generated by them to test it.
1. Verify the reboot-cause history title
2. Verify reboot-cause output are sorted in reverse chronological order

tests/common/reboot.py

@@ -1,8 +1,10 @@
 import threading
 import time
+import re
 import logging
 from multiprocessing.pool import ThreadPool, TimeoutError
 from errors import RunAnsibleModuleFail
+from collections import deque
 
 logger = logging.getLogger(__name__)
 
@@ -42,7 +44,7 @@
         "wait": 120,
         # We are searching two types of reboot cause.
         # This change relates to changes of PR #6130 in sonic-buildimage repository
-        "cause": r"'reboot'|Non-Hardware \(reboot",
+        "cause": r"'reboot'|Non-Hardware \(reboot|^reboot",
         "test_reboot_cause_only": False
     },
     REBOOT_TYPE_SOFT: {
@@ -75,6 +77,11 @@
     }
 }
 
+MAX_NUM_REBOOT_CAUSE_HISTORY = 10
+REBOOT_TYPE_HISTOYR_QUEUE = deque([], MAX_NUM_REBOOT_CAUSE_HISTORY)
+REBOOT_CAUSE_HISTORY_TITLE = ["name", "cause", "time", "user", "comment"]
+
+
 def get_warmboot_finalizer_state(duthost):
     try:
         res = duthost.command('systemctl is-active warmboot-finalizer.service',module_ignore_errors=True)
@@ -223,3 +230,46 @@ def check_reboot_cause(dut, reboot_cause_expected):
     reboot_cause_got = get_reboot_cause(dut)
     logging.debug("dut {} last reboot-cause {}".format(dut.hostname, reboot_cause_got))
     return reboot_cause_got == reboot_cause_expected
+
+
+def check_reboot_cause_history(dut, reboot_type_history_queue):
+    """
+    @summary: Check the reboot cause history on DUT. Can be used with wailt_until
+    @param dut: The AnsibleHost object of DUT.
+    @param reboot_type_history_queue: reboot type queue.
+    e.g.
+    show reboot-cause  history
+    Name                 Cause          Time                             User    Comment
+    -------------------  -------------  -------------------------------  ------  ---------
+    2021_09_09_14_15_13  Power Loss ()  N/A                              N/A     N/A
+    2021_09_09_14_06_17  reboot         Thu 09 Sep 2021 02:05:17 PM UTC  admin   N/A
+    2021_09_09_13_59_11  Watchdog ()    N/A                              N/A     N/A
+    2021_09_09_13_52_13  Power Loss ()  N/A                              N/A     N/A
+    2021_09_09_13_45_18  warm-reboot    Thu 09 Sep 2021 01:44:14 PM UTC  admin   N/A
+    2021_09_09_13_37_58  fast-reboot    Thu 09 Sep 2021 01:37:09 PM UTC  admin   N/A
+    2021_09_09_13_30_52  soft-reboot    Thu 09 Sep 2021 01:30:24 PM UTC  admin   N/A
+    2021_09_09_13_24_17  reboot         Thu 09 Sep 2021 01:23:17 PM UTC  admin   N/A
+    """
+    reboot_cause_history_got = dut.show_and_parse("show reboot-cause history")
+    logging.debug("dut {} reboot-cause history {}. reboot type history queue is {}".format(
+        dut.hostname, reboot_cause_history_got, reboot_type_history_queue))
+
+    logging.info("Verify reboot-cause history title")
+    if reboot_cause_history_got:
+        if not set(REBOOT_CAUSE_HISTORY_TITLE) == set(reboot_cause_history_got[0].keys()):
+            logging.error("Expected reboot-cause history title:{} not match actual reboot-cause history title:{}".format(
+                REBOOT_CAUSE_HISTORY_TITLE, reboot_cause_history_got[0].keys()))
+            return False
+
+    logging.info("Verify reboot-cause output are sorted in reverse chronological order" )
+    reboot_type_history_len = len(reboot_type_history_queue)
+    if reboot_type_history_len <= len(reboot_cause_history_got):
+        for index, reboot_type in enumerate(reboot_type_history_queue):
+            if not re.search(reboot_ctrl_dict[reboot_type]["cause"], reboot_cause_history_got[reboot_type_history_len-index-1]["cause"]):
+                logging.error("The {} reboot-cause not match. expected_reboot type={}, actual_reboot_cause={}".format(
+                    index, reboot_ctrl_dict[reboot_type]["cause"], reboot_cause_history_got[reboot_type_history_len-index]["cause"]))
+                return False
+        return True
+    logging.error("The number of expected reboot-cause:{} is more than that of actual reboot-cuase:{}".format(
+        reboot_type_history_len, len(reboot_type_history_queue)))
+    return False

tests/platform_tests/test_reboot.py

@@ -58,6 +58,7 @@ def reboot_and_check(localhost, dut, interfaces, xcvr_skip_list, reboot_type=REB
     logging.info("Run %s reboot on DUT" % reboot_type)
 
     reboot(dut, localhost, reboot_type=reboot_type, reboot_helper=reboot_helper, reboot_kwargs=reboot_kwargs)
+    REBOOT_TYPE_HISTOYR_QUEUE.append(reboot_type)
 
     check_interfaces_and_services(dut, interfaces, xcvr_skip_list, reboot_type)
 
@@ -77,6 +78,12 @@ def check_interfaces_and_services(dut, interfaces, xcvr_skip_list, reboot_type =
         assert wait_until(MAX_WAIT_TIME_FOR_REBOOT_CAUSE, 20, check_reboot_cause, dut, reboot_type), \
             "got reboot-cause failed after rebooted by %s" % reboot_type
 
+        if "201811" in dut.os_version or "201911" in dut.os_version:
+            logging.info("Skip check reboot-cause history for version before 202012")
+        else:
+            logger.info("Check reboot-cause history")
+            assert wait_until(MAX_WAIT_TIME_FOR_REBOOT_CAUSE, 20, check_reboot_cause_history, dut,
+                              REBOOT_TYPE_HISTOYR_QUEUE), "Check reboot-cause history failed after rebooted by %s" % reboot_type
         if reboot_ctrl_dict[reboot_type]["test_reboot_cause_only"]:
             logging.info("Further checking skipped for %s test which intends to verify reboot-cause only" % reboot_type)
             return