[dualtor] Implement dualtor T1 -> Standby Tor orchagent test cases. (s…

…onic-net#3110)

* Implement dualtor T1 -> Standby Tor test cases.

Signed-off-by: bingwang <[email protected]>

ansible/roles/test/files/ptftests/ip_in_ip_tunnel_test.py

@@ -2,7 +2,7 @@
 Description:    This file contains the IPinIP test for dualtor testbed
 
 Usage:          Examples of how to start this script
-                /usr/bin/ptf --test-dir ptftests ip_in_ip_tunnel_test.IpinIPTunnelTest --platform-dir ptftests --qlen=2000 --platform remote -t hash_key_list=['src-port', 'dst-port', 'src-mac', 'dst-mac', 'src-ip'];server_ip='192.168.0.2';active_tor_ip='10.1.0.33';standby_tor_mac='d4:af:f7:4d:af:18';standby_tor_ip='10.1.0.32';active_tor_mac='d4:af:f7:4d:a4:44';ptf_portchannel_indices={u'PortChannel0001': [29], u'PortChannel0003': [33], u'PortChannel0002': [31], u'PortChannel0004': [35]} --relax --debug info --log-file /tmp/ip_in_ip_tunnel_test.2021-02-10-07:14:46.log --socket-recv-size 16384
+                /usr/bin/ptf --test-dir ptftests ip_in_ip_tunnel_test.IpinIPTunnelTest --platform-dir ptftests --qlen=2000 --platform remote -t hash_key_list=['src-port', 'dst-port', 'src-mac', 'dst-mac', 'src-ip'];server_ip='192.168.0.2';active_tor_ip='10.1.0.33';standby_tor_mac='d4:af:f7:4d:af:18';standby_tor_ip='10.1.0.32';ptf_portchannel_indices={u'PortChannel0001': [29], u'PortChannel0003': [33], u'PortChannel0002': [31], u'PortChannel0004': [35]} --relax --debug info --log-file /tmp/ip_in_ip_tunnel_test.2021-02-10-07:14:46.log --socket-recv-size 16384
 
 '''
 #---------------------------------------------------------------------
@@ -39,12 +39,12 @@ def __init__(self):
         '''
         BaseTest.__init__(self)
         self.test_params = test_params_get()
+        self.logger = logging.getLogger("IPinIPTunnel")
 
     def setUp(self):
         self.server_ip = self.test_params['server_ip']
         self.server_port = int(self.test_params['server_port'])
         self.vlan_mac = self.test_params['vlan_mac']
-        self.active_tor_mac = self.test_params['active_tor_mac']
         self.standby_tor_mac = self.test_params['standby_tor_mac']
         self.active_tor_ip = self.test_params['active_tor_ip']
         self.standby_tor_ip = self.test_params['standby_tor_ip']
@@ -101,7 +101,7 @@ def generate_expected_packet(self, inner_pkt):
         """
         inner_pkt = inner_pkt.copy()
         inner_pkt.ttl = inner_pkt.ttl - 1
-        pkt = scapy.Ether(dst=self.active_tor_mac, src=self.standby_tor_mac) / \
+        pkt = scapy.Ether(dst="aa:aa:aa:aa:aa:aa", src=self.standby_tor_mac) / \
             scapy.IP(src=self.standby_tor_ip, dst=self.active_tor_ip) / inner_pkt[IP]
         exp_pkt = Mask(pkt)
         exp_pkt.set_do_not_care_scapy(scapy.Ether, 'dst')
@@ -154,11 +154,11 @@ def check_balance(self, pkt_distribution, hash_key):
         expect_packet_num = PACKET_NUM / portchannel_num
         pkt_num_lo = expect_packet_num * (1.0 - DIFF)
         pkt_num_hi = expect_packet_num * (1.0 + DIFF)
-        logging.info("hash key = {}".format(hash_key))
-        logging.info("%-10s \t %10s \t %10s \t" % ("port(s)", "exp_cnt", "act_cnt"))
+        self.logger.info("hash key = {}".format(hash_key))
+        self.logger.info("%-10s \t %10s \t %10s \t" % ("port(s)", "exp_cnt", "act_cnt"))
         balance = True
         for portchannel, count in pkt_distribution.items():
-            logging.info("%-10s \t %10s \t %10s \t" % (portchannel, str(expect_packet_num), str(count)))
+            self.logger.info("%-10s \t %10s \t %10s \t" % (portchannel, str(expect_packet_num), str(count)))
             if count < pkt_num_lo or count > pkt_num_hi:
                 balance = False
         if not balance:
@@ -181,12 +181,16 @@ def send_and_verify_packets(self):
                              port_id=self.server_port,
                              pkt=unexpected_packet,
                              timeout=TIMEOUT)
+
         # Step 2. verify packet is received from IPinIP tunnel and check balance
         for hash_key in self.hash_key_list:
+            self.logger.info("Verifying traffic balance for hash key {}".format(hash_key))
             pkt_distribution = {}
             for i in range(0, PACKET_NUM):
                 inner_pkt = self.generate_packet_to_server(hash_key)
                 tunnel_pkt = self.generate_expected_packet(inner_pkt)
+                self.logger.info("Sending packet dst_mac = {} src_mac = {} dst_ip = {} src_ip = {} from port {}" \
+                    .format(inner_pkt[Ether].dst, inner_pkt[Ether].src, inner_pkt[IP].dst, inner_pkt[IP].src, src_port))
                 send_packet(self, src_port, inner_pkt)
                 # Verify packet is received from IPinIP tunnel
                 idx, count = verify_packet_any_port(test=self,

tests/common/dualtor/dual_tor_mock.py

@@ -2,6 +2,7 @@
 import logging
 import os
 import pytest
+import time
 
 from ipaddress import ip_interface, IPv4Interface, IPv6Interface, \
                       ip_address, IPv4Address
@@ -278,10 +279,29 @@ def apply_peer_switch_table_to_dut(cleanup_mocked_configs, rand_selected_dut, mo
     peer_switch_key = 'PEER_SWITCH|{}'.format(peer_switch_hostname)
     device_meta_key = 'DEVICE_METADATA|localhost'
 
-    dut.shell('redis-cli -n 4 HSET "{}" "address_ipv4" "{}"'.format(peer_switch_key, mock_peer_switch_loopback_ip.ip))
-    dut.shell('redis-cli -n 4 HSET "{}" "{}" "{}"'.format(device_meta_key, 'subtype', 'dualToR'))
-    dut.shell('redis-cli -n 4 HSET "{}" "{}" "{}"'.format(device_meta_key, 'peer_switch', peer_switch_hostname))
+    cmds = ['redis-cli -n 4 HSET "{}" "address_ipv4" "{}"'.format(peer_switch_key, mock_peer_switch_loopback_ip.ip),
+            'redis-cli -n 4 HSET "{}" "{}" "{}"'.format(device_meta_key, 'subtype', 'DualToR'),
+            'redis-cli -n 4 HSET "{}" "{}" "{}"'.format(device_meta_key, 'peer_switch', peer_switch_hostname)]
+    dut.shell_cmds(cmds=cmds)
+    if dut.get_asic_name() == 'th2':
+        # Restart swss on TH2 platform
+        logger.info("Restarting swss service")
+        dut.shell('systemctl restart swss')
+        time.sleep(120)
+
+    yield
+    logger.info("Removing peer switch table")
 
+    cmds=['redis-cli -n 4 DEL "{}"'.format(peer_switch_key),
+          'redis-cli -n 4 HDEL"{}" "{}" "{}"'.format(device_meta_key, 'subtype', 'DualToR'),
+          'redis-cli -n 4 HDEL "{}" "{}" "{}"'.format(device_meta_key, 'peer_switch', peer_switch_hostname)]
+    dut.shell_cmds(cmds=cmds)
+    if dut.get_asic_name() == 'th2':
+        # Restart swss on TH2 platform
+        logger.info("Restarting swss service")
+        dut.shell('systemctl restart swss')
+        time.sleep(120)
+
     return
 
 
@@ -337,12 +357,16 @@ def apply_mux_cable_table_to_dut(cleanup_mocked_configs, rand_selected_dut, mock
     return
 
 
+def is_t0_mocked_dualtor(tbinfo):
+    return tbinfo["topo"]["type"] == "t0" and 'dualtor' not in tbinfo["topo"]["name"]
+
+
 @pytest.fixture(scope='module')
 def apply_mock_dual_tor_tables(request, tbinfo):
     '''
     Wraps all table fixtures for convenience
     '''
-    if tbinfo["topo"]["type"] == "t0" and 'dualtor' not in tbinfo["topo"]["name"]:
+    if is_t0_mocked_dualtor(tbinfo):
         request.getfixturevalue("apply_mux_cable_table_to_dut")
         request.getfixturevalue("apply_tunnel_table_to_dut")
         request.getfixturevalue("apply_peer_switch_table_to_dut")
@@ -354,7 +378,7 @@ def apply_mock_dual_tor_kernel_configs(request, tbinfo):
     '''
     Wraps all kernel related (routes and neighbor entries) fixtures for convenience
     '''
-    if tbinfo["topo"]["type"] == "t0" and 'dualtor' not in tbinfo["topo"]["name"]:
+    if is_t0_mocked_dualtor(tbinfo):
         request.getfixturevalue("apply_dual_tor_peer_switch_route")
         request.getfixturevalue("apply_dual_tor_neigh_entries")
         logger.info("Done applying kernel configs for dual ToR mock")
@@ -366,6 +390,6 @@ def cleanup_mocked_configs(duthost, tbinfo):
 
     yield
 
-    if tbinfo["topo"]["type"] == "t0" and 'dualtor' not in tbinfo["topo"]["name"]:
+    if is_t0_mocked_dualtor(tbinfo):
         logger.info("Load minigraph to reset the DUT %s", duthost.hostname)
         config_reload(duthost, config_source="minigraph")

tests/common/dualtor/dual_tor_utils.py

@@ -9,10 +9,10 @@
 from collections import defaultdict
 from natsort import natsorted
 from tests.common.config_reload import config_reload
-from tests.common.helpers.assertions import pytest_assert
 from tests.common.helpers.assertions import pytest_assert as pt_assert
 from tests.common.helpers.dut_ports import encode_dut_port_name
 from tests.common.dualtor.constants import UPPER_TOR, LOWER_TOR
+import ipaddress
 
 from ptf import mask
 from ptf import testutils
@@ -166,6 +166,34 @@ def get_t1_active_ptf_ports(dut, tbinfo):
 
     return ptf_portchannel_intfs
 
+def get_t1_bgp_up_ptf_ports(dut, tbinfo):
+    """
+    @summary: Get ptf port indices for PortChannels on which BGP session is up
+    @param dut: The DUT we are testing against
+    @param tbinfo: The fixture tbinfo
+    @return: A dict { "PortChannel0001": [0, 1], ...}
+    """
+    config_facts = dut.get_running_config_facts()
+    mg_facts = dut.get_extended_minigraph_facts(tbinfo)
+    bgp_facts = dut.bgp_facts()['ansible_facts']
+    ip_interfaces = dut.shell('show ip interface')['stdout_lines'][2:]
+    portchannels = []
+    for k, v in bgp_facts['bgp_neighbors'].items():
+        if v['state'] == 'established':
+            for line in ip_interfaces:
+                if k in line:
+                    portchannels.append(line.split()[0])
+                    break
+
+    ptf_portchannel_intfs = {}
+    for k, v in config_facts['PORTCHANNEL'].items():
+        if k in portchannels:
+            ptf_portchannel_intfs[k]  = []
+            for member in v['members']:
+                ptf_portchannel_intfs[k].append(mg_facts['minigraph_ptf_indices'][member])
+
+    return ptf_portchannel_intfs
+
 
 def update_mux_configs_and_config_reload(dut, state):
     """
@@ -176,10 +204,10 @@ def update_mux_configs_and_config_reload(dut, state):
     @param state: A str, auto|active|standby
     """
     STATE_LIST = ['auto', 'active', 'standby']
-    pytest_assert(state in STATE_LIST, "state should be one of {}".format(STATE_LIST))
+    pt_assert(state in STATE_LIST, "state should be one of {}".format(STATE_LIST))
 
     mux_cable_config = dut.shell("sonic-cfggen -d  --var-json 'MUX_CABLE'")['stdout']
-    pytest_assert(len(mux_cable_config.strip()) != 0, "No mux_cable configuration is found in config_db")
+    pt_assert(len(mux_cable_config.strip()) != 0, "No mux_cable configuration is found in config_db")
 
     # Update mux_cable state and dump to a temp file
     mux_cable_config_json = json.loads(mux_cable_config)
@@ -289,7 +317,7 @@ def _shutdown_fanout_tor_intfs(tor_host, tor_fanouthosts, tbinfo, dut_intfs=None
             Defaults to None.
 
     Returns:
-        dict (fanouthost: list): Each key is a fanout host, and the corresponding value is the interfaces that were shut down 
+        dict (fanouthost: list): Each key is a fanout host, and the corresponding value is the interfaces that were shut down
                                  on that host device.
     """
     if not dut_intfs:
@@ -563,14 +591,13 @@ def mux_cable_server_ip(dut):
     return json.loads(mux_cable_config)
 
 
-def check_tunnel_balance(ptfhost, active_tor_mac, standby_tor_mac, vlan_mac, active_tor_ip, standby_tor_ip, targer_server_ip, target_server_port, ptf_portchannel_indices):
+def check_tunnel_balance(ptfhost, standby_tor_mac, vlan_mac, active_tor_ip, standby_tor_ip, target_server_ip, target_server_port, ptf_portchannel_indices):
     """
     Function for testing traffic distribution among all avtive T1.
     A test script will be running on ptf to generate traffic to standby interface, and the traffic will be forwarded to
     active ToR. The running script will capture all traffic and verify if these packets are distributed evenly.
     Args:
         ptfhost: The ptf host connected to current testbed
-        active_tor_mac: MAC address of active ToR
         standby_tor_mac: MAC address of the standby ToR
         vlan_mac: MAC address of Vlan (For verifying packet)
         active_tor_ip: IP Address of Loopback0 of active ToR (For verifying packet)
@@ -583,9 +610,8 @@ def check_tunnel_balance(ptfhost, active_tor_mac, standby_tor_mac, vlan_mac, act
     """
     HASH_KEYS = ["src-port", "dst-port", "src-ip"]
     params = {
-        "server_ip": targer_server_ip,
+        "server_ip": target_server_ip,
         "server_port": target_server_port,
-        "active_tor_mac": active_tor_mac,
         "standby_tor_mac": standby_tor_mac,
         "vlan_mac": vlan_mac,
         "active_tor_ip": active_tor_ip,
@@ -674,6 +700,51 @@ def get_crm_nexthop_counter(host):
     return crm_facts['resources']['ipv4_nexthop']['used']
 
 
+def dualtor_info(ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo):
+    """
+    @summary: A helper function for collecting info of dualtor testbed.
+    @param ptfhost: The ptf host fixture
+    @param rand_selected_dut: The randomly selected dut host, will be set as standby ToR
+    @param rand_unselected_dut: The other dut in dualtor testbed, will be set as active ToR
+    @param tbinfo: The tbinfo fixture
+    @return: A dict, can be used as the argument of check_tunnel_balance
+    """
+    active_tor = rand_unselected_dut
+    standby_tor = rand_selected_dut
+    standby_tor_mg_facts = standby_tor.get_extended_minigraph_facts(tbinfo)
+
+    def _get_iface_ip(mg_facts, ifacename):
+        for loopback in mg_facts['minigraph_lo_interfaces']:
+            if loopback['name'] == ifacename and ipaddress.ip_address(loopback['addr']).version == 4:
+                return loopback['addr']
+
+    res = {}
+    res['ptfhost'] = ptfhost
+    res['standby_tor_mac'] = standby_tor.facts['router_mac']
+    vlan_name = standby_tor_mg_facts['minigraph_vlans'].keys()[0]
+    res['vlan_mac'] = standby_tor.get_dut_iface_mac(vlan_name)
+    res['standby_tor_ip'] = _get_iface_ip(standby_tor_mg_facts, 'Loopback0')
+
+    if 't0' in tbinfo["topo"]["name"]:
+        # For mocked dualtor
+        res['active_tor_ip'] = str(ipaddress.ip_address(res['standby_tor_ip']) + 1)
+        # For mocked dualtor, routes to peer switch is static 
+        res['ptf_portchannel_indices'] = get_t1_active_ptf_ports(standby_tor, tbinfo)
+    else:
+        active_tor_mg_facts = active_tor.get_extended_minigraph_facts(tbinfo)
+        res['active_tor_ip'] = _get_iface_ip(active_tor_mg_facts, 'Loopback0')
+        res['ptf_portchannel_indices'] = get_t1_bgp_up_ptf_ports(standby_tor, tbinfo)
+
+    servers = mux_cable_server_ip(standby_tor)
+    random_server_iface = random.choice(servers.keys())
+
+    res['target_server_ip'] = servers[random_server_iface]['server_ipv4'].split('/')[0]
+    res['target_server_port'] = standby_tor_mg_facts['minigraph_ptf_indices'][random_server_iface]
+
+    logger.debug("dualtor info is generated {}".format(res))
+    return res
+
+
 def show_arp(duthost, neighbor_addr):
     """Show arp table entry for neighbor."""
     command = "/usr/sbin/arp -n %s" % neighbor_addr

tests/common/fixtures/ptfhost_utils.py

@@ -7,7 +7,6 @@
 from jinja2 import Template
 from natsort import natsorted
 
-
 logger = logging.getLogger(__name__)
 
 ROOT_DIR = "/root"
@@ -200,13 +199,22 @@ def run_icmp_responder(duthost, ptfhost, tbinfo):
     ptfhost.shell("supervisorctl stop icmp_responder")
 
 
-@pytest.fixture(scope='module')
-def run_garp_service(duthost, ptfhost, tbinfo, change_mac_addresses):
-
+@pytest.fixture(scope='module', autouse=True)
+def run_garp_service(duthost, ptfhost, tbinfo, change_mac_addresses, mock_server_base_ip_addr, tor_mux_intfs):
     garp_config = {}
 
     ptf_indices = duthost.get_extended_minigraph_facts(tbinfo)["minigraph_ptf_indices"]
-    mux_cable_table = duthost.get_running_config_facts()['MUX_CABLE']
+    if 't0' in tbinfo['topo']['name']:
+        # For mocked dualtor testbed
+        mux_cable_table = {}
+        server_ipv4_base_addr, _ = mock_server_base_ip_addr
+        for i, intf in enumerate(tor_mux_intfs):
+            server_ipv4 = str(server_ipv4_base_addr + i)
+            mux_cable_table[intf] = {}
+            mux_cable_table[intf]['server_ipv4'] = unicode(server_ipv4)
+    else:
+        # For physical dualtor testbed
+        mux_cable_table = duthost.get_running_config_facts()['MUX_CABLE']
 
     logger.info("Generating GARP service config file")
 

tests/dualtor/conftest.py

@@ -2,7 +2,7 @@
 import logging
 import time
 
-from tests.common.dualtor.dual_tor_utils import get_crm_nexthop_counter, lower_tor_host # lgtm[py/unused-import]
+from tests.common.dualtor.dual_tor_utils import get_crm_nexthop_counter # lgtm[py/unused-import]
 from tests.common.helpers.assertions import pytest_assert as py_assert
 from tests.common.fixtures.ptfhost_utils import run_garp_service
 
@@ -12,27 +12,32 @@
 
 
 @pytest.fixture
-def set_crm_polling_interval(lower_tor_host):
+def set_crm_polling_interval(rand_selected_dut):
     """
     A function level fixture to set crm polling interval to 1 second
     """
     wait_time = 2
-    lower_tor_host.command("crm config polling interval {}".format(CRM_POLL_INTERVAL))
+    logging.info("Setting crm polling interval to {} seconds".format(CRM_POLL_INTERVAL))
+    rand_selected_dut.command("crm config polling interval {}".format(CRM_POLL_INTERVAL))
     logging.info("Waiting {} sec for CRM counters to become updated".format(wait_time))
     time.sleep(wait_time)
     yield
-    lower_tor_host.command("crm config polling interval {}".format(CRM_DEFAULT_POLL_INTERVAL))
+    logging.info("Setting crm polling interval to {} seconds".format(CRM_DEFAULT_POLL_INTERVAL))
+    rand_selected_dut.command("crm config polling interval {}".format(CRM_DEFAULT_POLL_INTERVAL))
 
 
 @pytest.fixture
-def verify_crm_nexthop_counter_not_increased(lower_tor_host):
+def verify_crm_nexthop_counter_not_increased(rand_selected_dut, set_crm_polling_interval):
     """
     A function level fixture to verify crm nexthop counter not increased
     """
-    original_counter = get_crm_nexthop_counter(lower_tor_host)
+    original_counter = get_crm_nexthop_counter(rand_selected_dut)
+    logging.info("Before test: crm nexthop counter = {}".format(original_counter))
     yield
-    diff = get_crm_nexthop_counter(lower_tor_host) - original_counter
-    py_assert(diff == 0, "crm nexthop counter is increased by {}.".format(diff))
+    time.sleep(CRM_POLL_INTERVAL)
+    diff = get_crm_nexthop_counter(rand_selected_dut) - original_counter
+    logging.info("Before test: crm nexthop counter = {}".format(original_counter + diff))
+    py_assert(diff <= 0, "crm nexthop counter is increased by {}.".format(diff))
 
 
 def pytest_addoption(parser):
@@ -50,6 +55,7 @@ def pytest_addoption(parser):
         help="The number of iterations for mux stress test"
     )
 
+
 @pytest.fixture(scope="module", autouse=True)
 def common_setup_teardown(request, tbinfo):
     if 'dualtor' in tbinfo['topo']['name']: