From 178f64ab3eb2f204d2277a5e06ee911078fc16bc Mon Sep 17 00:00:00 2001 From: bingwang Date: Tue, 30 Mar 2021 01:00:45 -0700 Subject: [PATCH 1/2] Add server down test case Signed-off-by: bingwang --- tests/common/dualtor/dual_tor_utils.py | 17 +++++++++ tests/dualtor/test_server_failure.py | 52 ++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 tests/dualtor/test_server_failure.py diff --git a/tests/common/dualtor/dual_tor_utils.py b/tests/common/dualtor/dual_tor_utils.py index 6f0b941e55..de7bf4562f 100644 --- a/tests/common/dualtor/dual_tor_utils.py +++ b/tests/common/dualtor/dual_tor_utils.py @@ -643,3 +643,20 @@ def rand_selected_interface(rand_selected_dut): iface = str(random.choice(server_ips.keys())) logging.info("select DUT interface %s to test.", iface) return iface, server_ips[iface] + + +def show_muxcable_status(duthost): + """ + Show muxcable status and parse into a dict + """ + command = "show muxcable status" + output = duthost.shell(command)["stdout_lines"] + + ret = {} + for i in range(2, len(output)): + port, status, health = output[i].split() + ret[port] = {'status': status, 'health': health} + + return ret + + \ No newline at end of file diff --git a/tests/dualtor/test_server_failure.py b/tests/dualtor/test_server_failure.py new file mode 100644 index 0000000000..08e55236e2 --- /dev/null +++ b/tests/dualtor/test_server_failure.py @@ -0,0 +1,52 @@ +import pytest +import time +from tests.common.dualtor.mux_simulator_control import toggle_simulator_port_to_upper_tor, simulator_flap_counter, simulator_server_down +from tests.common.helpers.assertions import pytest_assert, pytest_require +from tests.common.dualtor.dual_tor_utils import show_muxcable_status, rand_selected_interface +from tests.common.fixtures.ptfhost_utils import change_mac_addresses, run_garp_service, run_icmp_responder + +pytestmark = [ + pytest.mark.topology('t0'), + pytest.mark.usefixtures('run_garp_service', 'run_icmp_responder') +] + + +def test_server_down(duthosts, tbinfo, rand_selected_interface, simulator_flap_counter, simulator_server_down, toggle_simulator_port_to_upper_tor, loganalyzer): + """ + Verify that mux cable is not toggled excessively. + """ + pytest_require('dualtor' in tbinfo['topo']['name'], + "Only run on dualtor testbed") + + for analyzer in list(loganalyzer.values()): + analyzer.ignore_regex.append(r".*ERR swss#orchagent: :- setState: State transition from active to active is not-handled") + + upper_tor = duthosts[tbinfo['duts'][0]] + lower_tor = duthosts[tbinfo['duts'][1]] + PAUSE_TIME = 5 + + itfs, _ = rand_selected_interface + # Set upper_tor as active + toggle_simulator_port_to_upper_tor(itfs) + time.sleep(PAUSE_TIME) + mux_flap_counter_0 = simulator_flap_counter(itfs) + # Server down + simulator_server_down(itfs) + time.sleep(PAUSE_TIME) + # Verify mux_cable state on upper_tor is active + mux_state_upper_tor = show_muxcable_status(upper_tor) + pytest_assert(mux_state_upper_tor[itfs]['status'] == 'active' and mux_state_upper_tor[itfs]['health'] == 'unhealthy', + "mux_cable status is unexpected. Should be (active, unhealthy)") + # Verify mux_cable state on lower_tor is standby + mux_state_lower_tor = show_muxcable_status(lower_tor) + pytest_assert(mux_state_lower_tor[itfs]['status'] == 'standby' and mux_state_lower_tor[itfs]['health'] == 'unhealthy', + "mux_cable status is unexpected. Should be (standby, unhealthy)") + # Verify that mux_cable flap_counter should be no larger than 3 + # lower_tor(standby) -> active -> standby + # upper_tor(active) -> active + # The toggle from both tor may be overlapped and invisible + mux_flap_counter_1 = simulator_flap_counter(itfs) + pytest_assert(mux_flap_counter_1 - mux_flap_counter_0 <= 3, + "The mux_cable flap count should be no larger than 3 ({})".format(mux_flap_counter_1 - mux_flap_counter_0)) + + \ No newline at end of file From 3b859fef8e390d996991ef3567fe341001b9a0b0 Mon Sep 17 00:00:00 2001 From: bingwang Date: Tue, 30 Mar 2021 01:07:14 -0700 Subject: [PATCH 2/2] Add empty line Signed-off-by: bingwang --- tests/common/dualtor/dual_tor_utils.py | 1 - tests/dualtor/test_server_failure.py | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/common/dualtor/dual_tor_utils.py b/tests/common/dualtor/dual_tor_utils.py index de7bf4562f..c4c36e975e 100644 --- a/tests/common/dualtor/dual_tor_utils.py +++ b/tests/common/dualtor/dual_tor_utils.py @@ -659,4 +659,3 @@ def show_muxcable_status(duthost): return ret - \ No newline at end of file diff --git a/tests/dualtor/test_server_failure.py b/tests/dualtor/test_server_failure.py index 08e55236e2..a002044926 100644 --- a/tests/dualtor/test_server_failure.py +++ b/tests/dualtor/test_server_failure.py @@ -49,4 +49,3 @@ def test_server_down(duthosts, tbinfo, rand_selected_interface, simulator_flap_c pytest_assert(mux_flap_counter_1 - mux_flap_counter_0 <= 3, "The mux_cable flap count should be no larger than 3 ({})".format(mux_flap_counter_1 - mux_flap_counter_0)) - \ No newline at end of file