[multi-dut] Making loganalyzer fixture work with multiple DUTs in multi-dut testbed

tests/autorestart/test_container_autorestart.py

@@ -28,7 +28,7 @@ def config_reload_after_tests(duthost):
     config_reload(duthost)
 
 @pytest.fixture(autouse=True)
-def ignore_expected_loganalyzer_exception(loganalyzer, enum_dut_feature):
+def ignore_expected_loganalyzer_exception(duthost, loganalyzer, enum_dut_feature):
     """
         Ignore expected failure/error messages during testing the autorestart feature.
 
@@ -95,9 +95,9 @@ def ignore_expected_loganalyzer_exception(loganalyzer, enum_dut_feature):
     _, feature = decode_dut_port_name(enum_dut_feature)
 
     if loganalyzer:
-        loganalyzer.ignore_regex.extend(ignore_regex_dict['common'])
+        loganalyzer[duthost.hostname].ignore_regex.extend(ignore_regex_dict['common'])
         if feature in ignore_regex_dict:
-            loganalyzer.ignore_regex.extend(ignore_regex_dict[feature])
+            loganalyzer[duthost.hostname].ignore_regex.extend(ignore_regex_dict[feature])
 
 
 def get_group_program_info(duthost, container_name, group_name):

tests/common/plugins/loganalyzer/__init__.py

@@ -3,6 +3,7 @@
 
 from loganalyzer import LogAnalyzer
 from tests.common.errors import RunAnsibleModuleFail
+import re
 
 
 def pytest_addoption(parser):
@@ -11,33 +12,40 @@ def pytest_addoption(parser):
 
 
 @pytest.fixture(autouse=True)
-def loganalyzer(duthosts, rand_one_dut_hostname, request):
-    duthost = duthosts[rand_one_dut_hostname]
+def loganalyzer(duthosts, request):
     if request.config.getoption("--disable_loganalyzer") or "disable_loganalyzer" in request.keywords:
         logging.info("Log analyzer is disabled")
         yield
         return
 
-    # Force rotate logs
-    try:
-        duthost.shell(
-            "/usr/sbin/logrotate -f /etc/logrotate.conf > /dev/null 2>&1"
-            )
-    except RunAnsibleModuleFail as e:
-        logging.warning("logrotate is failed. Command returned:\n"
-                        "Stdout: {}\n"
-                        "Stderr: {}\n"
-                        "Return code: {}".format(e.results["stdout"], e.results["stderr"], e.results["rc"]))
-
-    loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix=request.node.name)
-    logging.info("Add start marker into DUT syslog")
-    marker = loganalyzer.init()
-    logging.info("Load config and analyze log")
-    # Read existed common regular expressions located with legacy loganalyzer module
-    loganalyzer.load_common_config()
-
-    yield loganalyzer
+    analyzers = {}
+    markers = {}
+    # Analyze all the duts
+    for duthost in duthosts:
+        # Force rotate logs
+        try:
+            duthost.shell(
+                "/usr/sbin/logrotate -f /etc/logrotate.conf > /dev/null 2>&1"
+                )
+        except RunAnsibleModuleFail as e:
+            logging.warning("logrotate is failed. Command returned:\n"
+                            "Stdout: {}\n"
+                            "Stderr: {}\n"
+                            "Return code: {}".format(e.results["stdout"], e.results["stderr"], e.results["rc"]))
+
+        loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix=request.node.name)
+        logging.info("Add start marker into DUT syslog")
+        marker = loganalyzer.init()
+        logging.info("Load config and analyze log")
+        # Read existed common regular expressions located with legacy loganalyzer module
+        loganalyzer.load_common_config()
+        analyzers[duthost.hostname] = loganalyzer
+        markers[duthost.hostname] = marker
+
+    yield analyzers
+
     # Skip LogAnalyzer if case is skipped
     if "rep_call" in request.node.__dict__ and request.node.rep_call.skipped:
         return
-    loganalyzer.analyze(marker)
+    for dut_hostname, dut_analyzer in analyzers.items():
+        dut_analyzer.analyze(markers[dut_hostname])

tests/copp/test_copp.py

@@ -131,7 +131,7 @@ def copp_testbed(
         _teardown_testbed(duthost, creds, ptfhost, test_params)
 
 @pytest.fixture(autouse=True)
-def ignore_expected_loganalyzer_exceptions(loganalyzer):
+def ignore_expected_loganalyzer_exceptions(rand_one_dut_hostname, loganalyzer):
     """
         Ignore expected failures logs during test execution.
 
@@ -152,7 +152,7 @@ def ignore_expected_loganalyzer_exceptions(loganalyzer):
     ]
 
     if loganalyzer:  # Skip if loganalyzer is disabled
-        loganalyzer.ignore_regex.extend(ignoreRegex)
+        loganalyzer[rand_one_dut_hostname].ignore_regex.extend(ignoreRegex)
 
 def _copp_runner(dut, ptf, protocol, test_params, dut_type):
     """

tests/dhcp_relay/test_dhcp_relay.py

@@ -16,13 +16,13 @@
 DEFAULT_DHCP_CLIENT_PORT = 68
 
 @pytest.fixture(autouse=True)
-def ignore_expected_loganalyzer_exceptions(loganalyzer):
+def ignore_expected_loganalyzer_exceptions(rand_one_dut_hostname, loganalyzer):
     """Ignore expected failures logs during test execution."""
     if loganalyzer:
         ignoreRegex = [
             ".*ERR snmp#snmp-subagent.*",
         ]
-        loganalyzer.ignore_regex.extend(ignoreRegex)
+        loganalyzer[rand_one_dut_hostname].ignore_regex.extend(ignoreRegex)
 
     yield
 

tests/drop_packets/drop_packets.py

@@ -170,16 +170,16 @@ def rif_port_down(duthosts, rand_one_dut_hostname, setup, fanouthosts, loganalyz
 
     fanout_neighbor, fanout_intf = fanout_switch_port_lookup(fanouthosts, duthost.hostname, rif_member_iface)
 
-    loganalyzer.expect_regex = [LOG_EXPECT_PORT_OPER_DOWN_RE.format(rif_member_iface)]
-    with loganalyzer as _:
+    loganalyzer[rand_one_dut_hostname].expect_regex = [LOG_EXPECT_PORT_OPER_DOWN_RE.format(rif_member_iface)]
+    with loganalyzer[rand_one_dut_hostname] as _:
         fanout_neighbor.shutdown(fanout_intf)
 
     time.sleep(1)
 
     yield ip_dst
 
-    loganalyzer.expect_regex = [LOG_EXPECT_PORT_OPER_UP_RE.format(rif_member_iface)]
-    with loganalyzer as _:
+    loganalyzer[rand_one_dut_hostname].expect_regex = [LOG_EXPECT_PORT_OPER_UP_RE.format(rif_member_iface)]
+    with loganalyzer[rand_one_dut_hostname] as _:
         fanout_neighbor.no_shutdown(fanout_intf)
         time.sleep(wait_after_ports_up)
 

tests/drop_packets/test_drop_counters.py

@@ -74,14 +74,14 @@ def acl_setup(duthosts, rand_one_dut_hostname, loganalyzer):
 
     logger.info("Applying {}".format(dut_conf_file_path))
 
-    loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_CREATE_RE]
-    with loganalyzer as analyzer:
+    loganalyzer[rand_one_dut_hostname].expect_regex = [LOG_EXPECT_ACL_RULE_CREATE_RE]
+    with loganalyzer[rand_one_dut_hostname] as analyzer:
         duthost.command("config acl update full {}".format(dut_conf_file_path))
 
     yield
 
-    loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_REMOVE_RE]
-    with loganalyzer as analyzer:
+    loganalyzer[rand_one_dut_hostname].expect_regex = [LOG_EXPECT_ACL_RULE_REMOVE_RE]
+    with loganalyzer[rand_one_dut_hostname] as analyzer:
         logger.info("Applying {}".format(dut_clear_conf_file_path))
         duthost.command("config acl update full {}".format(dut_clear_conf_file_path))
         logger.info("Removing {}".format(dut_tmp_dir))

tests/k8s/conftest.py

@@ -19,7 +19,7 @@ def check_k8s_vms(k8scluster):
 
 
 @pytest.fixture(autouse=True)
-def ignore_expected_loganalyzer_exceptions(loganalyzer):
+def ignore_expected_loganalyzer_exceptions(duthost, loganalyzer):
     """
        Ignore expected failures logs during test execution
 
@@ -35,5 +35,5 @@ def ignore_expected_loganalyzer_exceptions(loganalyzer):
              ".*for troubleshooting tips.*",
              ".*kubeproxy.*",
          ]
-         loganalyzer.ignore_regex.extend(ignoreRegex)
+         loganalyzer[duthost.hostname].ignore_regex.extend(ignoreRegex)
     yield

tests/lldp/test_lldp.py

@@ -29,15 +29,15 @@ def test_lldp(duthosts, rand_one_dut_hostname, localhost, collect_techsupport):
         assert v['chassis']['name'] == config_facts['DEVICE_NEIGHBOR'][k]['name']
         # Compare the LLDP neighbor interface with minigraph neigbhor interface (exclude the management port)
         assert v['port']['ifname'] == config_facts['DEVICE_NEIGHBOR'][k]['port']
-   
+
 
 def test_lldp_neighbor(duthosts, rand_one_dut_hostname, localhost, eos,
                        collect_techsupport, loganalyzer):
     """ verify LLDP information on neighbors """
     duthost = duthosts[rand_one_dut_hostname]
 
     if loganalyzer:
-        loganalyzer.ignore_regex.extend([
+        loganalyzer[rand_one_dut_hostname].ignore_regex.extend([
             ".*ERR syncd#syncd: :- check_fdb_event_notification_data.*",
             ".*ERR syncd#syncd: :- process_on_fdb_event: invalid OIDs in fdb \
                 notifications, NOT translating and NOT storing in ASIC DB.*",

tests/pc/test_po_cleanup.py

@@ -9,7 +9,7 @@
 ]
 
 @pytest.fixture(autouse=True)
-def ignore_expected_loganalyzer_exceptions(loganalyzer):
+def ignore_expected_loganalyzer_exceptions(rand_one_dut_hostname, loganalyzer):
     """
         Ignore expected failures logs during test execution.
 
@@ -25,12 +25,12 @@ def ignore_expected_loganalyzer_exceptions(loganalyzer):
         ignoreRegex = [
             ".*",
         ]
-        loganalyzer.ignore_regex.extend(ignoreRegex)
+        loganalyzer[rand_one_dut_hostname].ignore_regex.extend(ignoreRegex)
         expectRegex = [
             ".*teamd#teammgrd: :- cleanTeamProcesses.*",
             ".*teamd#teamsyncd: :- cleanTeamSync.*"
         ]
-        loganalyzer.expect_regex.extend(expectRegex)
+        loganalyzer[rand_one_dut_hostname].expect_regex.extend(expectRegex)
 
 
 def check_kernel_po_interface_cleaned(duthost):

tests/pc/test_po_update.py

@@ -11,7 +11,7 @@
 ]
 
 @pytest.fixture(autouse=True)
-def ignore_expected_loganalyzer_exceptions(loganalyzer):
+def ignore_expected_loganalyzer_exceptions(rand_one_dut_hostname, loganalyzer):
     """
         Ignore expected failures logs during test execution.
 
@@ -29,7 +29,7 @@ def ignore_expected_loganalyzer_exceptions(loganalyzer):
             ".*ERR syncd#syncd: :- process_on_fdb_event: FDB notification was not sent since it contain invalid OIDs, bug.*",
             ".*ERR syncd#syncd: :- translate_vid_to_rid: unable to get RID for VID.*",
         ]
-        loganalyzer.ignore_regex.extend(ignoreRegex)
+        loganalyzer[rand_one_dut_hostname].ignore_regex.extend(ignoreRegex)
 
     yield
 

tests/pfcwd/test_pfcwd_timer_accuracy.py

@@ -39,7 +39,7 @@ def ignore_loganalyzer_exceptions(rand_one_dut_hostname, loganalyzer):
             ".*ERR syncd#syncd: :- process_on_fdb_event: invalid OIDs in fdb notifications, NOT translating and NOT storing in ASIC DB.*",
             ".*ERR syncd#syncd: :- process_on_fdb_event: FDB notification was not sent since it contain invalid OIDs, bug.*"
         ]
-        loganalyzer.ignore_regex.extend(ignoreRegex)
+        loganalyzer[rand_one_dut_hostname].ignore_regex.extend(ignoreRegex)
 
     yield
 

tests/qos/qos_sai_base.py

@@ -500,7 +500,7 @@ def updateDockerService(host, docker="", action="", service=""):
             updateDockerService(duthost, action="start", **service)
 
     @pytest.fixture(autouse=True)
-    def updateLoganalyzerExceptions(self, loganalyzer):
+    def updateLoganalyzerExceptions(self, rand_one_dut_hostname, loganalyzer):
         """
             Update loganalyzer ignore regex list
 
@@ -530,7 +530,7 @@ def updateLoganalyzerExceptions(self, loganalyzer):
                 ".*ERR monit.*'bgp\|bgpmon' status failed.*'/usr/bin/python.* /usr/local/bin/bgpmon' is not running.*",
                 ".*ERR monit.*bgp\|fpmsyncd.*status failed.*NoSuchProcess process no longer exists.*",
             ]
-            loganalyzer.ignore_regex.extend(ignoreRegex)
+            loganalyzer[rand_one_dut_hostname].ignore_regex.extend(ignoreRegex)
 
         yield
 

tests/route/test_route_perf.py

@@ -20,7 +20,7 @@
 ROUTE_TABLE_NAME = 'ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY'
 
 @pytest.fixture(autouse=True)
-def ignore_expected_loganalyzer_exceptions(loganalyzer):
+def ignore_expected_loganalyzer_exceptions(rand_one_dut_hostname, loganalyzer):
     """
         Ignore expected failures logs during test execution.
 
@@ -38,7 +38,7 @@ def ignore_expected_loganalyzer_exceptions(loganalyzer):
     ]
     if loganalyzer:
         # Skip if loganalyzer is disabled
-        loganalyzer.ignore_regex.extend(ignoreRegex)
+        loganalyzer[rand_one_dut_hostname].ignore_regex.extend(ignoreRegex)
 
 @pytest.fixture(params=[4, 6])
 def ip_versions(request):