Merge pull request #3398 from mab68/tt14

Re-add fault-tolerance tests

.travis.yml

@@ -72,6 +72,11 @@ jobs:
       language: bash
       services:
         - docker
+    - stage: test
+      env: MATRIX_SHARD=fault-tolerance
+      language: bash
+      services:
+        - docker
     - stage: deploy
       language: python
       python: 3.6

clib/clib_mininet_test.py

@@ -14,4 +14,4 @@
 import clib_mininet_tests
 
 if __name__ == '__main__':
-    test_main(clib_mininet_tests.__name__)
+    test_main([clib_mininet_tests.__name__])

clib/clib_mininet_test_main.py

@@ -628,7 +628,7 @@ def clean_test_dirs(root_tmpdir, all_successful, sanity, keep_logs, dumpfail):
                         dump_failed_test(test_name, test_dir)
 
 
-def run_tests(module, hw_config, requested_test_classes, dumpfail, debug,
+def run_tests(modules, hw_config, requested_test_classes, dumpfail, debug,
               keep_logs, serial, repeat, excluded_test_classes, report_json_filename,
               port_order):
     """Actually run the test suites, potentially in parallel."""
@@ -650,38 +650,42 @@ def run_tests(module, hw_config, requested_test_classes, dumpfail, debug,
     if keep_logs:
         resultclass = FaucetResult
     all_successful = False
+    no_tests = True
+
+    for module in modules:
+        sanity_tests, single_tests, parallel_tests = expand_tests(
+            module, requested_test_classes, excluded_test_classes,
+            hw_config, root_tmpdir, ports_sock, serial, port_order)
+
+        if (sanity_tests.countTestCases() + single_tests.countTestCases() + parallel_tests.countTestCases()):
+            no_tests = False
+            sanity_result = run_sanity_test_suite(root_tmpdir, resultclass, sanity_tests)
+            if sanity_result.wasSuccessful():
+                while True:
+                    all_successful = run_test_suites(debug, report_json_filename,
+                                                     hw_config, root_tmpdir, resultclass,
+                                                     copy.deepcopy(single_tests),
+                                                     copy.deepcopy(parallel_tests),
+                                                     sanity_result)
+                    if not repeat:
+                        break
+                    if not all_successful:
+                        break
+                    print('repeating run')
+            else:
+                report_results([sanity_result], hw_config, report_json_filename)
 
-    sanity_tests, single_tests, parallel_tests = expand_tests(
-        module, requested_test_classes, excluded_test_classes,
-        hw_config, root_tmpdir, ports_sock, serial, port_order)
-
-    if (sanity_tests.countTestCases() + single_tests.countTestCases() + parallel_tests.countTestCases()):
-        sanity_result = run_sanity_test_suite(root_tmpdir, resultclass, sanity_tests)
-        if sanity_result.wasSuccessful():
-            while True:
-                all_successful = run_test_suites(debug, report_json_filename,
-                                                 hw_config, root_tmpdir, resultclass,
-                                                 copy.deepcopy(single_tests),
-                                                 copy.deepcopy(parallel_tests),
-                                                 sanity_result)
-                if not repeat:
-                    break
-                if not all_successful:
-                    break
-                print('repeating run')
-        else:
-            report_results([sanity_result], hw_config, report_json_filename)
-
+    if no_tests:
+        print('no tests selected')
+        shutil.rmtree(root_tmpdir)
+        sys.exit(0)
+    else:
         decoded_pcap_logs = glob.glob(os.path.join(
             os.path.join(root_tmpdir, '*'), '*of.cap.txt'))
         pipeline_superset_report(decoded_pcap_logs)
         clean_test_dirs(
             root_tmpdir, all_successful,
             sanity_result.wasSuccessful(), keep_logs, dumpfail)
-    else:
-        print('no tests selected')
-        shutil.rmtree(root_tmpdir)
-        sys.exit(0)
 
     if not all_successful:
         sys.exit(-1)
@@ -722,6 +726,8 @@ def parse_args():
         '-x', help='list of test classes to exclude')
     parser.add_argument(
         '-r', '--repeat', action='store_true', help='repeat tests until failure')
+    parser.add_argument(
+        '-t', '--tolerance', default=False, action='store_false', help='run fault-tolerance tests')
 
     excluded_test_classes = []
     report_json_filename = None
@@ -753,10 +759,10 @@ def parse_args():
         args.loglevel, args.profile)
 
 
-def test_main(module):
+def test_main(modules):
     """Test main."""
 
-    print('testing module %s' % module)
+    print('testing module %s' % modules)
 
     (requested_test_classes, clean, dumpfail, debug, keep_logs, nocheck,
      serial, repeat, excluded_test_classes, report_json_filename, port_order,
@@ -788,7 +794,7 @@ def test_main(module):
         pr.enable()
 
     run_tests(
-        module, hw_config, requested_test_classes, dumpfail, debug,
+        modules, hw_config, requested_test_classes, dumpfail, debug,
         keep_logs, serial, repeat, excluded_test_classes, report_json_filename, port_order)
 
     if profile:

clib/mininet_test_base.py

@@ -160,6 +160,8 @@ def switches_name_ordered(self):
 
     def first_switch(self):
         """Return first switch by name order."""
+        if not self.switches_name_ordered():
+            return None
         return self.switches_name_ordered()[0]
 
     def rand_dpid(self):
@@ -412,28 +414,34 @@ def setUp(self):
     def hostns(self, host):
         return '%s' % host.name
 
+    def dump_switch_flows(self, switch):
+        """Dump switch information to tmpdir"""
+        for dump_cmd in (
+                'dump-flows', 'dump-groups', 'dump-meters',
+                'dump-group-stats', 'dump-ports', 'dump-ports-desc',
+                'meter-stats'):
+            switch_dump_name = os.path.join(self.tmpdir, '%s-%s.log' % (switch.name, dump_cmd))
+            # TODO: occasionally fails with socket error.
+            switch.cmd('%s %s %s > %s' % (self.OFCTL, dump_cmd, switch.name, switch_dump_name),
+                       success=None)
+        for other_cmd in ('show', 'list controller', 'list manager'):
+            other_dump_name = os.path.join(self.tmpdir, '%s.log' % other_cmd.replace(' ', ''))
+            switch.cmd('%s %s > %s' % (self.VSCTL, other_cmd, other_dump_name))
+
     def tearDown(self, ignore_oferrors=False):
         """Clean up after a test.
            ignore_oferrors: return OF errors rather than failing"""
         if self.NETNS:
             for host in self.hosts_name_ordered()[:1]:
                 if self.get_host_netns(host):
                     self.quiet_commands(host, ['ip netns del %s' % self.hostns(host)])
-        self.first_switch().cmd('ip link > %s' % os.path.join(self.tmpdir, 'ip-links.log'))
+        first_switch = self.first_switch()
+        if first_switch:
+            self.first_switch().cmd('ip link > %s' % os.path.join(self.tmpdir, 'ip-links.log'))
         switch_names = []
         for switch in self.net.switches:
             switch_names.append(switch.name)
-            for dump_cmd in (
-                    'dump-flows', 'dump-groups', 'dump-meters',
-                    'dump-group-stats', 'dump-ports', 'dump-ports-desc',
-                    'meter-stats'):
-                switch_dump_name = os.path.join(self.tmpdir, '%s-%s.log' % (switch.name, dump_cmd))
-                # TODO: occasionally fails with socket error.
-                switch.cmd('%s %s %s > %s' % (self.OFCTL, dump_cmd, switch.name, switch_dump_name),
-                           success=None)
-            for other_cmd in ('show', 'list controller', 'list manager'):
-                other_dump_name = os.path.join(self.tmpdir, '%s.log' % other_cmd.replace(' ', ''))
-                switch.cmd('%s %s > %s' % (self.VSCTL, other_cmd, other_dump_name))
+            self.dump_switch_flows(switch)
             switch.cmd('%s del-br %s' % (self.VSCTL, switch.name))
         self._stop_net()
         self.net = None