[Kubernetes Testing] Add DUT Join Kubernetes Master Test Cases (#2520)

Added four Kubernetes DUT Join Test Cases: 

- TC_JOIN_2: Ensure kube server disable flag works as expected- DUT joins master when disable=false, and DUT resets from master when disable=true
- TC_JOIN_3: Ensure DUT joins after config reload with no config change (config saved with disable=false)
- TC_JOIN_4: Ensure DUT resets from master when config reload toggles disable to true (config saved with disable=true)
- TC_JOIN_5: Ensure DUT joins master when config reload toggles disable to false (config saved with disable=false)

ansible/group_vars/all/creds.yml

@@ -12,4 +12,4 @@ sonic_default_passwords:
 sonic_password: "password"
 
 k8s_master_login: "ubuntu"
-k8s_master_password: use_own_value
+k8s_master_password: "use_own_value"

tests/common/devices.py

@@ -1201,7 +1201,7 @@ def ensure_kubelet_running(self):
                 self.shell("sudo systemctl start kubelet")
 
 
-class K8sMasterCluster():
+class K8sMasterCluster(object):
     """
     @summary: Class that encapsulates Kubernetes master cluster
 
@@ -1222,7 +1222,8 @@ def __init__(self, k8smasters):
             else:
                 self.backend_masters.append(k8smaster)
 
-    def get_master_vip(self):
+    @property
+    def vip(self):
         """
         @summary: Retrieve VIP of Kubernetes master cluster
 

tests/k8s/k8s_test_utilities.py

@@ -5,9 +5,6 @@
 
 logger = logging.getLogger(__name__)
 
-WAIT_FOR_SYNC = 60 # time unit seconds
-
-
 def join_master(duthost, master_vip):
     """
     Joins DUT to Kubernetes master
@@ -70,7 +67,7 @@ def clean_vip_iptables_rules(duthost, master_vip):
 def check_connected(duthost):
     """
     Checks if the DUT already shows status 'connected' to Kubernetes master
-
+    
     Args:
         duthost: DUT host object
     
@@ -104,7 +101,7 @@ def poll_for_status_change(duthost, exp_status, poll_wait_secs=5, min_wait_time=
     timeout_wait_secs = max_wait_time - min_wait_time
     while (timeout_wait_secs > 0):
        if (check_connected(duthost) == exp_status):
-           logging.info("Time taken to update status: {} seconds".format(timeout_wait_secs))
+           logging.info("Time taken to update Kube server status: {} seconds".format(timeout_wait_secs))
            return True
        time.sleep(poll_wait_secs)
        timeout_wait_secs -= poll_wait_secs

tests/k8s/test_config_reload.py

@@ -0,0 +1,102 @@
+import pytest
+import time
+import k8s_test_utilities as ku
+
+from tests.common.helpers.assertions import pytest_assert
+from tests.common.platform.processes_utils import wait_critical_processes
+
+pytestmark = [
+    pytest.mark.topology('any')
+]
+
+def test_config_reload_no_toggle(duthost, k8scluster):
+    """
+    Test case to ensure that when DUT starts as joined to master, and config is saved with disable=false, DUT is still joined to master after config reload
+    
+    Joins master
+
+    Performs config reload
+
+    Ensures that DUT is still joined to master after config reload
+
+    Args:
+        duthost: DUT host object
+        k8scluster: shortcut fixture for getting cluster of Kubernetes master hosts
+    """
+    ku.join_master(duthost, k8scluster.vip) # Assertion within to ensure successful join
+    dut_cmds = ['sudo config save -y',
+                'sudo config reload -y']
+    duthost.shell_cmds(cmds=dut_cmds)
+    wait_critical_processes(duthost)
+
+    server_connect_exp_status = True
+    server_connect_act_status = ku.check_connected(duthost)
+    server_connect_status_updated = ku.poll_for_status_change(duthost, server_connect_exp_status)
+    pytest_assert(server_connect_status_updated, "Unexpected k8s server connection status after config reload, Expected server connected status: {}, Found server connected status: {}".format(server_connect_exp_status, server_connect_act_status))
+
+
+def test_config_reload_toggle_join(duthost, k8scluster):
+    """
+    Test case to ensure that when DUT is not joined to the master due to (unsaved) disable=true, but config is saved with disable=false, DUT joins after config reload
+
+    Saves config with configured VIP and disable=false
+
+    Sets disable=true without saving config, and ensure that DUT resets from master
+
+    Performs config reload
+
+    Ensures that DUT is joined to master after config reload
+
+    Args:
+        duthost: DUT host object
+        k8scluster: shortcut fixture for getting cluster of Kubernetes master hosts
+    """
+    dut_cmds = ['sudo config kube server ip {}'.format(k8scluster.vip),
+                'sudo config kube server disable off',
+                'sudo config save -y']
+    duthost.shell_cmds(cmds=dut_cmds)
+
+    duthost.shell('sudo config kube server disable on')
+    server_connect_exp_status = False
+    server_connect_act_status = ku.check_connected(duthost)
+    server_connect_status_updated = ku.poll_for_status_change(duthost, server_connect_exp_status)
+    pytest_assert(server_connect_status_updated, "Unexpected k8s server connection status after setting disable=true, Expected server connected status: {}, Found server connected status: {}".format(server_connect_exp_status, server_connect_act_status))
+
+    duthost.shell('sudo config reload -y')
+    wait_critical_processes(duthost)
+
+    server_connect_exp_status = True
+    server_connect_act_status = ku.check_connected(duthost)
+    server_connect_status_updated = ku.poll_for_status_change(duthost, server_connect_exp_status)
+    pytest_assert(server_connect_status_updated, "Unexpected k8s server connection status after config reload, Expected server connected status: {}, Found server connected status: {}".format(server_connect_exp_status, server_connect_act_status))
+
+
+def test_config_reload_toggle_reset(duthost, k8scluster):
+    """
+    Test case to ensure that when DUT is joined to master (disable=false, unsaved) but config is saved with disable=true, DUT resets from master after config reload
+
+    Saves config with disable=true
+
+    Joins master, which sets disable=false unsaved
+
+    Performs config reload
+
+    Ensures that DUT has reset from the master after config reload, as disable=true was saved 
+    
+    Args:
+        duthost: DUT host object
+        k8scluster: shortcut fixture for getting cluster of Kubernetes master hosts
+    """
+    dut_cmds = ['sudo config kube server disable on',
+                'sudo config save -y']
+    duthost.shell_cmds(cmds=dut_cmds)
+
+    ku.join_master(duthost, k8scluster.vip) 
+
+    duthost.shell('sudo config reload -y')
+    wait_critical_processes(duthost)
+
+    server_connect_exp_status = False
+    server_connect_act_status = ku.check_connected(duthost)
+    server_connect_status_updated = ku.poll_for_status_change(duthost, server_connect_exp_status)
+    pytest_assert(server_connect_status_updated, "Unexpected k8s server connection status after config reload, Expected server connected status: {}, Found server connected status: {}".format(server_connect_exp_status, server_connect_act_status))

tests/k8s/test_disable_flag.py

@@ -0,0 +1,37 @@
+import pytest
+import time
+import k8s_test_utilities as ku
+
+from tests.common.helpers.assertions import pytest_assert
+
+pytestmark = [
+    pytest.mark.topology('any')
+]
+
+def test_disable_flag(duthost, k8scluster):
+    """
+    Test case to ensure that kube server disable flag works as expected when toggled
+
+    Joins master to set baseline state (disable=false, joined to master)
+
+    Set disable=true, ensure DUT resets from master
+
+    Set disable=false, ensure DUT joins master
+
+    Args:
+        duthost: DUT host object
+        k8scluster: shortcut fixture for getting cluster of Kubernetes master hosts
+    """
+    ku.join_master(duthost, k8scluster.vip)
+
+    duthost.shell('sudo config kube server disable on')
+    server_connect_exp_status = False
+    server_connect_act_status = ku.check_connected(duthost)
+    server_connect_status_updated = ku.poll_for_status_change(duthost, server_connect_exp_status)
+    pytest_assert(server_connect_status_updated, "Test disable flag failed, Expected server connected status: {}, Found server connected status: {}".format(server_connect_exp_status, server_connect_act_status))
+
+    duthost.shell('sudo config kube server disable off')
+    server_connect_exp_status = True
+    server_connect_act_status = ku.check_connected(duthost)
+    server_connect_status_updated = ku.poll_for_status_change(duthost, server_connect_exp_status)
+    pytest_assert(server_connect_status_updated, "Test disable flag failed, Expected server connected status: {}, Found server connected status: {}".format(server_connect_exp_status, server_connect_act_status))

tests/k8s/test_join_available_master.py

@@ -1,15 +1,14 @@
 import pytest
-import logging
 import time
 import k8s_test_utilities as ku
 
 from tests.common.helpers.assertions import pytest_assert
 
 WAIT_FOR_SYNC = 60
 
-logger = logging.getLogger(__name__)
-
-
+pytestmark = [
+    pytest.mark.topology('any')
+]
 
 def test_join_available_master(duthost, k8scluster):
     """
@@ -25,24 +24,23 @@ def test_join_available_master(duthost, k8scluster):
         duthost: DUT host object
         k8scluster: shortcut fixture for getting cluster of Kubernetes master hosts
     """
-    master_vip = k8scluster.get_master_vip()
     duthost.shell('sudo config kube server disable on')
     time.sleep(WAIT_FOR_SYNC)
 
     server_connect_exp_status = False
     server_connect_act_status = ku.check_connected(duthost)
     pytest_assert(server_connect_exp_status == server_connect_act_status, "DUT shows unexpected kubernetes server connected status, Expected server connected status: {}, Found server connected status: {}".format(server_connect_exp_status, server_connect_act_status))
 
-    ku.make_vip_unreachable(duthost, master_vip)
+    ku.make_vip_unreachable(duthost, k8scluster.vip)
     duthost.shell('sudo config kube server disable off') 
-    duthost.shell('sudo config kube server ip {}'.format(master_vip))
+    duthost.shell('sudo config kube server ip {}'.format(k8scluster.vip))
     time.sleep(WAIT_FOR_SYNC)
 
     server_connect_exp_status = False
     server_connect_act_status = ku.check_connected(duthost)
     pytest_assert(server_connect_exp_status == server_connect_act_status, "DUT shows unexpected kubernetes server connected status, Expected server connected status: {}, Found server connected status: {}".format(server_connect_exp_status, server_connect_act_status))
 
-    ku.make_vip_reachable(duthost, master_vip)
+    ku.make_vip_reachable(duthost, k8scluster.vip)
 
     server_connect_exp_status = True
     server_connect_act_status = ku.check_connected(duthost)