stability/verify unification

tools/ci/check_stability.py

@@ -20,7 +20,7 @@
 from tools import localpaths
 
 logger = None
-stability_run, write_inconsistent, write_results = None, None, None
+run_step, write_inconsistent, write_results = None, None, None
 wptrunner = None
 
 def setup_logging():
@@ -35,10 +35,9 @@ def setup_logging():
 
 
 def do_delayed_imports():
-    global stability_run, write_inconsistent, write_results, wptrunner
-    from tools.wpt.stability import run as stability_run
-    from tools.wpt.stability import write_inconsistent, write_results
+    global wptrunner, run_step, write_inconsistent, write_results
     from wptrunner import wptrunner
+    from wptrunner.stability import run_step, write_inconsistent, write_results
 
 
 class TravisFold(object):
@@ -266,11 +265,15 @@ def run(venv, wpt_args, **kwargs):
 
         do_delayed_imports()
 
-        wpt_kwargs["stability"] = True
         wpt_kwargs["prompt"] = False
         wpt_kwargs["install_browser"] = True
         wpt_kwargs["install"] = wpt_kwargs["product"].split(":")[0] == "firefox"
 
+        wpt_kwargs["pause_after_test"] = False
+        wpt_kwargs["verify_log_full"] = True
+        if wpt_kwargs["repeat"] == 1:
+            wpt_kwargs["repeat"] = 10
+
         wpt_kwargs = setup_wptrunner(venv, **wpt_kwargs)
 
         logger.info("Using binary %s" % wpt_kwargs["binary"])
@@ -279,9 +282,8 @@ def run(venv, wpt_args, **kwargs):
     with TravisFold("running_tests"):
         logger.info("Starting tests")
 
-
         wpt_logger = wptrunner.logger
-        iterations, results, inconsistent = stability_run(venv, wpt_logger, **wpt_kwargs)
+        results, inconsistent, iterations = run_step(wpt_logger, wpt_kwargs["repeat"], True, {}, **wpt_kwargs)
 
     if results:
         if inconsistent:

tools/wpt/run.py

@@ -49,8 +49,6 @@ def create_parser():
                         help="Browser to run tests in")
     parser.add_argument("--yes", "-y", dest="prompt", action="store_false", default=True,
                         help="Don't prompt before installing components")
-    parser.add_argument("--stability", action="store_true",
-                        help="Stability check tests")
     parser.add_argument("--install-browser", action="store_true",
                         help="Install the latest development version of the browser")
     parser._add_container_actions(wptcommandline.create_parser())
@@ -434,28 +432,14 @@ def setup_wptrunner(venv, prompt=True, install=False, **kwargs):
 def run(venv, **kwargs):
     #Remove arguments that aren't passed to wptrunner
     prompt = kwargs.pop("prompt", True)
-    stability = kwargs.pop("stability", True)
     install_browser = kwargs.pop("install_browser", False)
 
     kwargs = setup_wptrunner(venv,
                              prompt=prompt,
                              install=install_browser,
                              **kwargs)
 
-    if stability:
-        import stability
-        iterations, results, inconsistent = stability.run(venv, logger, **kwargs)
-
-        def log(x):
-            print(x)
-
-        if inconsistent:
-            stability.write_inconsistent(log, inconsistent, iterations)
-        else:
-            log("All tests stable")
-        rv = len(inconsistent) > 0
-    else:
-        rv = run_single(venv, **kwargs) > 0
+    rv = run_single(venv, **kwargs) > 0
 
     return rv
 

tools/wptrunner/wptrunner/stability.py

@@ -181,8 +181,8 @@ def run_step(logger, iterations, restart_after_iteration, kwargs_extras, **kwarg
     kwargs.update(kwargs_extras)
 
     def wrap_handler(x):
-        x = LogLevelFilter(x, "WARNING")
         if not kwargs["verify_log_full"]:
+            x = LogLevelFilter(x, "WARNING")
             x = LogActionFilter(x, ["log", "process_output"])
         return x
 

tools/wpt/tests/test_stability.py → tools/wptrunner/wptrunner/tests/test_stability.py

@@ -1,4 +1,9 @@
-from tools.wpt import stability
+import sys
+from os.path import dirname, join
+
+sys.path.insert(0, join(dirname(__file__), "..", ".."))
+
+from wptrunner import stability
 
 def test_is_inconsistent():
     assert stability.is_inconsistent({"PASS": 10}, 10) is False

tools/wptrunner/wptrunner/wptcommandline.py

@@ -81,9 +81,13 @@ def create_parser(product_choices=None):
     mode_group.add_argument("--list-tests", action="store_true",
                             default=False,
                             help="List all tests that will run")
-    mode_group.add_argument("--verify", action="store_true",
-                            default=False,
-                            help="Run a stability check on the selected tests")
+    stability_group = mode_group.add_mutually_exclusive_group()
+    stability_group.add_argument("--verify", action="store_true",
+                                 default=False,
+                                 help="Run a stability check on the selected tests")
+    stability_group.add_argument("--stability", action="store_true",
+                                 default=False,
+                                 help=argparse.SUPPRESS)
     mode_group.add_argument("--verify-log-full", action="store_true",
                             default=False,
                             help="Output per-iteration test results when running verify")

tools/wptrunner/wptrunner/wptrunner.py

@@ -300,6 +300,14 @@ def run_tests(config, test_paths, product, **kwargs):
 
 def check_stability(**kwargs):
     import stability
+    if kwargs["stability"]:
+        logger.warning("--stability is deprecated; please use --verify instead!")
+        kwargs['verify_max_time'] = None
+        kwargs['verify_chaos_mode'] = False
+        kwargs['verify_repeat_loop'] = 0
+        kwargs['verify_repeat_restart'] = 10 if kwargs['repeat'] == 1 else kwargs['repeat']
+        kwargs['verify_output_results'] = True
+
     return stability.check_stability(logger,
                                      max_time=kwargs['verify_max_time'],
                                      chaos_mode=kwargs['verify_chaos_mode'],
@@ -315,7 +323,7 @@ def start(**kwargs):
         list_disabled(**kwargs)
     elif kwargs["list_tests"]:
         list_tests(**kwargs)
-    elif kwargs["verify"]:
+    elif kwargs["verify"] or kwargs["stability"]:
         check_stability(**kwargs)
     else:
         return not run_tests(**kwargs)