Add github checks output support to stability check

tools/ci/taskcluster-run.py

@@ -84,6 +84,9 @@ def main(product, commit_range, wpt_args):
     command = ["python", "./wpt", "run"] + wpt_args + [product]
 
     logger.info("Executing command: %s" % " ".join(command))
+    with open("/home/test/artifacts/checkrun.md", "a") as f:
+        f.write("\n**WPT Command:** `%s`\n\n" % " ".join(command))
+
     retcode = subprocess.call(command, env=dict(os.environ, TERM="dumb"))
     if retcode != 0:
         sys.exit(retcode)

tools/ci/tc/decision.py

@@ -241,6 +241,8 @@ def create_tc_task(event, task, taskgroup_id, depends_on_ids, env_extra=None):
         },
         "routes": ["checks"]
     }
+    if "extra" in task:
+        task_data["extra"].update(task["extra"])
     if env_extra:
         task_data["payload"]["env"].update(env_extra)
     if depends_on_ids:

tools/ci/tc/github_checks_output.py

@@ -0,0 +1,29 @@
+class GitHubChecksOutputter(object):
+    """Provides a method to output data to be shown in the GitHub Checks UI.
+
+    This can be useful to provide a summary of a given check (e.g. the lint)
+    to enable developers to quickly understand what has gone wrong. The output
+    supports markdown format.
+
+    See https://docs.taskcluster.net/docs/reference/integrations/github/checks#custom-text-output-in-checks
+    """
+    def __init__(self, path):
+        self.path = path
+
+    def output(self, line):
+        with open(self.path, 'a') as f:
+            f.write(line)
+            f.write('\n')
+
+
+__outputter = None
+def get_gh_checks_outputter(kwargs):
+    """Return the outputter for GitHub Checks output, if enabled.
+
+    :param kwargs: The arguments passed to the program (to look for the
+                   --github_checks_text_file flag)
+    """
+    global __outputter
+    if kwargs['github_checks_text_file'] and __outputter is None:
+        __outputter = GitHubChecksOutputter(kwargs['github_checks_text_file'])
+    return __outputter

tools/ci/tc/tasks/test.yml

@@ -10,6 +10,13 @@ components:
       public/results:
         path: /home/test/artifacts
         type: directory
+    extra:
+      github:
+        customCheckRun:
+          # We set both textArtifactName and annotationsArtifactName due
+          # to https://github.com/taskcluster/taskcluster/issues/3191
+          textArtifactName: public/results/checkrun.md
+          annotationsArtifactName: public/results/checkrun.md
 
   wpt-testharness:
     chunks: 16
@@ -296,6 +303,7 @@ tasks:
               --channel=${vars.channel}
               --verify
               --verify-no-chaos-mode
+              --github-checks-text-file="/home/test/artifacts/checkrun.md"
 
         - wpt-${vars.browser}-${vars.channel}-results:
             use:

tools/wptrunner/wptrunner/stability.py

@@ -13,6 +13,7 @@
 
 here = os.path.dirname(__file__)
 localpaths = imp.load_source("localpaths", os.path.abspath(os.path.join(here, os.pardir, os.pardir, "localpaths.py")))
+from ci.tc.github_checks_output import get_gh_checks_outputter
 from wpt.markdown import markdown_adjust, table
 
 
@@ -178,8 +179,32 @@ def err_string(results_dict, iterations):
     return rv
 
 
+def write_github_checks_summary_inconsistent(log, inconsistent, iterations):
+    """Outputs a summary of inconsistent tests for GitHub Checks."""
+    log("Some affected tests had inconsistent (flaky) results:\n")
+    write_inconsistent(log, inconsistent, iterations)
+    log("\n")
+    log("These may be pre-existing or new flakes. Please try to reproduce (see "
+        "the above WPT command, though some flags may not be needed when "
+        "running locally) and determine if your change introduced the flake. "
+        "If you are unable to reproduce the problem, please tag "
+        "`@web-platform-tests/wpt-core-team` in a comment for help.\n")
+
+
+def write_github_checks_summary_slow_tests(log, slow):
+    """Outputs a summary of slow tests for GitHub Checks."""
+    log("Some affected tests had slow results:\n")
+    write_slow_tests(log, slow)
+    log("\n")
+    log("These may be pre-existing or newly slow tests. Slow tests indicate "
+        "that a test ran very close to the test timeout limit and so may "
+        "become TIMEOUT-flaky in the future. Consider speeding up the test or "
+        "breaking it into multiple tests. For help, please tag "
+        "`@web-platform-tests/wpt-core-team` in a comment.\n")
+
+
 def write_inconsistent(log, inconsistent, iterations):
-    """Output inconsistent tests to logger.error."""
+    """Output inconsistent tests to the passed in logging function."""
     log("## Unstable results ##\n")
     strings = [(
         "`%s`" % markdown_adjust(test),
@@ -191,6 +216,7 @@ def write_inconsistent(log, inconsistent, iterations):
 
 
 def write_slow_tests(log, slow):
+    """Output slow tests to the passed in logging function."""
     log("## Slow tests ##\n")
     strings = [(
         "`%s`" % markdown_adjust(test),
@@ -321,6 +347,8 @@ def check_stability(logger, repeat_loop=10, repeat_restart=5, chaos_mode=True, m
     start_time = datetime.now()
     step_results = []
 
+    github_checks_outputter = get_gh_checks_outputter(kwargs)
+
     for desc, step_func in steps:
         if max_time and datetime.now() - start_time > max_time:
             logger.info("::: Test verification is taking too long: Giving up!")
@@ -337,12 +365,16 @@ def check_stability(logger, repeat_loop=10, repeat_restart=5, chaos_mode=True, m
 
         if inconsistent:
             step_results.append((desc, "FAIL"))
+            if github_checks_outputter:
+                write_github_checks_summary_inconsistent(github_checks_outputter.output, inconsistent, iterations)
             write_inconsistent(logger.info, inconsistent, iterations)
             write_summary(logger, step_results, "FAIL")
             return 1
 
         if slow:
             step_results.append((desc, "FAIL"))
+            if github_checks_outputter:
+                write_github_checks_summary_slow_tests(github_checks_outputter.output, slow)
             write_slow_tests(logger.info, slow)
             write_summary(logger, step_results, "FAIL")
             return 1

tools/wptrunner/wptrunner/wptcommandline.py

@@ -348,6 +348,11 @@ def create_parser(product_choices=None):
                              help="Command-line argument to forward to the "
                                   "Sauce Connect binary (repeatable)")
 
+    taskcluster_group = parser.add_argument_group("Taskcluster-specific")
+    taskcluster_group.add_argument("--github-checks-text-file",
+                                   dest="github_checks_text_file",
+                                   help="Path to GitHub checks output file")
+
     webkit_group = parser.add_argument_group("WebKit-specific")
     webkit_group.add_argument("--webkit-port", dest="webkit_port",
                               help="WebKit port")