Validate pull requests in TaskCluster

.taskcluster.yml

@@ -2,42 +2,101 @@ version: 1
 policy:
   pullRequests: collaborators
 tasks:
-  $if: tasks_for == "github-push"
-  then:
-    $if: event.ref == "refs/heads/master"
-    then:
-      $flattenDeep:
+  $flattenDeep:
+    - $if: tasks_for == "github-push"
+      then:
+        $if: event.ref == "refs/heads/master"
+        then:
+          $map: [{name: firefox, channel: nightly}, {name: chrome, channel: dev}]
+          each(browser):
+            $map:
+              - [testharness, 1, 15]
+              - [testharness, 2, 15]
+              - [testharness, 3, 15]
+              - [testharness, 4, 15]
+              - [testharness, 5, 15]
+              - [testharness, 6, 15]
+              - [testharness, 7, 15]
+              - [testharness, 8, 15]
+              - [testharness, 9, 15]
+              - [testharness, 10, 15]
+              - [testharness, 11, 15]
+              - [testharness, 12, 15]
+              - [testharness, 13, 15]
+              - [testharness, 14, 15]
+              - [testharness, 15, 15]
+              - [reftest, 1, 10]
+              - [reftest, 2, 10]
+              - [reftest, 3, 10]
+              - [reftest, 4, 10]
+              - [reftest, 5, 10]
+              - [reftest, 6, 10]
+              - [reftest, 7, 10]
+              - [reftest, 8, 10]
+              - [reftest, 9, 10]
+              - [reftest, 10, 10]
+              - [wdspec, 1, 1]
+            each(chunk):
+              taskId: {$eval: 'as_slugid(browser.name + browser.channel + chunk[0] + str(chunk[1]))'}
+              taskGroupId: {$eval: 'as_slugid("task group")'}
+              created: {$fromNow: ''}
+              deadline: {$fromNow: '24 hours'}
+              provisionerId: aws-provisioner-v1
+              # Contributors interested in configurating TaskCluster to run
+              # against their fork of WPT should change the `workerType` to
+              # "github-worker".
+              workerType: wpt-docker-worker
+              metadata:
+                name: wpt-${browser.name}-${browser.channel}-${chunk[0]}-${chunk[1]}
+                description: >-
+                  A subset of WPT's "${chunk[0]}" tests (chunk number ${chunk[1]}
+                  of ${chunk[2]}), run in the ${browser.channel} release of
+                  ${browser.name}.
+                owner: ${event.pusher.email}
+                source: ${event.repository.url}
+              payload:
+                image: gsnedders/web-platform-tests:0.13
+                maxRunTime: 7200
+                artifacts:
+                  public/results:
+                    path: /home/test/artifacts
+                    type: directory
+                command:
+                  - /bin/bash
+                  - --login
+                  - -c
+                  - set -ex;
+                    ~/start.sh
+                      ${event.repository.url}
+                      ${event.ref[len('refs/heads/'):]}
+                      ${event.after}
+                      ${browser.name}-${browser.channel};
+                    cd ~/web-platform-tests;
+                    ./tools/ci/taskcluster-run.py
+                      ${browser.name}
+                      --
+                      --log-wptreport=../artifacts/wpt_report.json
+                      --no-fail-on-unexpected
+                      --test-type=${chunk[0]}
+                      --this-chunk=${chunk[1]}
+                      --total-chunks=${chunk[2]};
+    - $if: tasks_for == "github-pull-request"
+      then:
         $map: [{name: firefox, channel: nightly}, {name: chrome, channel: dev}]
         each(browser):
           $map:
-            - [testharness, 1, 15]
-            - [testharness, 2, 15]
-            - [testharness, 3, 15]
-            - [testharness, 4, 15]
-            - [testharness, 5, 15]
-            - [testharness, 6, 15]
-            - [testharness, 7, 15]
-            - [testharness, 8, 15]
-            - [testharness, 9, 15]
-            - [testharness, 10, 15]
-            - [testharness, 11, 15]
-            - [testharness, 12, 15]
-            - [testharness, 13, 15]
-            - [testharness, 14, 15]
-            - [testharness, 15, 15]
-            - [reftest, 1, 10]
-            - [reftest, 2, 10]
-            - [reftest, 3, 10]
-            - [reftest, 4, 10]
-            - [reftest, 5, 10]
-            - [reftest, 6, 10]
-            - [reftest, 7, 10]
-            - [reftest, 8, 10]
-            - [reftest, 9, 10]
-            - [reftest, 10, 10]
-            - [wdspec, 1, 1]
-          each(chunk):
-            taskId: {$eval: 'as_slugid(browser.name + browser.channel + chunk[0] + str(chunk[1]))'}
+            - name: wpt-${browser.name}-${browser.channel}-stability
+              description: >-
+                Verify that all tests affected by a pull request are stable
+                when executed in ${browser.name}.
+              extra_args: '--verify'
+            - name: wpt-${browser.name}-${browser.channel}-results
+              description: >-
+                Collect results for all tests affected by a pull request in
+                ${browser.name}.
+              extra_args: '--no-fail-on-unexpected --log-wptreport=../artifacts/wpt_report.json'
+          each(operation):
+            taskId: {$eval: 'as_slugid(operation.name)'}
             taskGroupId: {$eval: 'as_slugid("task group")'}
             created: {$fromNow: ''}
             deadline: {$fromNow: '24 hours'}
@@ -47,12 +106,9 @@ tasks:
             # "github-worker".
             workerType: wpt-docker-worker
             metadata:
-              name: wpt-${browser.name}-${browser.channel}-${chunk[0]}-${chunk[1]}
-              description: >-
-                A subset of WPT's "${chunk[0]}" tests (chunk number ${chunk[1]}
-                of ${chunk[2]}), run in the ${browser.channel} release of
-                ${browser.name}.
-              owner: ${event.pusher.email}
+              name: ${operation.name}
+              description: ${operation.description}
+              owner: ${event.pull_request.user.login}@users.noreply.github.com
               source: ${event.repository.url}
             payload:
               image: gsnedders/web-platform-tests:0.13
@@ -65,13 +121,22 @@ tasks:
                 - /bin/bash
                 - --login
                 - -c
-                - "~/start.sh ${event.repository.url} ${event.ref[len('refs/heads/'):]} ${event.after} ${browser.name}-${browser.channel} &&
-                  cd ~/web-platform-tests &&
-                  ./tools/ci/ci_taskcluster.sh ${browser.name} ${chunk[0]} ${chunk[1]} ${chunk[2]}"
-    else: []
-  # > NOTE: A well-designed template should produce `tasks: []` for any
-  # > unrecognized `task_for` values; this allows later expansion of this
-  # > service to handle more events.
-  #
-  # https://docs.taskcluster.net/docs/reference/integrations/taskcluster-github/docs/taskcluster-yml-v1
-  else: []
+                - set -ex;
+                  ~/start.sh
+                    ${event.repository.clone_url}
+                    ${event.pull_request.base.ref}
+                    ${event.pull_request.base.sha}
+                    ${browser.name}-${browser.channel};
+                  cd ~/web-platform-tests;
+                  git fetch
+                    ${event.repository.clone_url}
+                    refs/pull/${event.number}/head;
+                  git checkout FETCH_HEAD;
+                  result=0;
+                  ./tools/ci/taskcluster-run.py
+                    --commit-range ${event.pull_request.base.sha}...
+                    ${browser.name}
+                    --
+                    ${operation.extra_args} || result=$?;
+                  echo $result > ../artifacts/run-return-code.txt;
+                  echo "Command exited with code $result (failures are allowed while this task is being vetted)."

dom/events/CustomEvent.html

@@ -4,6 +4,10 @@
 <script src="/resources/testharnessreport.js"></script>
 <div id="log"></div>
 <script>
+test(function() {
+  assert_true(Math.random() > 0.5);
+}, 'intentionally unstable test');
+
 test(function() {
   var type = "foo";
 

tools/ci/taskcluster-run.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+
+import argparse
+import gzip
+import logging
+import os
+import shutil
+import subprocess
+
+browser_specific_args = {
+    "firefox": ["--install-browser", "--reftest-internal"]
+}
+
+def tests_affected(commit_range):
+    output = subprocess.check_output([
+        "python", "./wpt", "tests-affected", "--null", commit_range
+    ], stderr=open(os.devnull, "w"))
+
+    tests = output.split("\0")
+
+    # Account for trailing null byte
+    if not tests[-1]:
+        tests.pop()
+
+    return tests
+
+
+def find_wptreport(args):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--log-wptreport', action='store')
+    return parser.parse_known_args(args)[0].log_wptreport
+
+
+def gzip_file(filename):
+    with open(filename, 'rb') as f_in:
+        with gzip.open('%s.gz' % filename, 'wb') as f_out:
+            shutil.copyfileobj(f_in, f_out)
+
+
+def main(product, commit_range, wpt_args):
+    """Invoke the `wpt run` command according to the needs of the TaskCluster
+    continuous integration service."""
+
+    logger = logging.getLogger("tc-run")
+    logger.setLevel(logging.INFO)
+    handler = logging.StreamHandler()
+    handler.setFormatter(
+        logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+    )
+    logger.addHandler(handler)
+
+    child = subprocess.Popen(['python', './wpt', 'manifest-download'])
+    child.wait()
+
+    if commit_range:
+        logger.info(
+            "Identifying tests affected in range '%s'..." % commit_range
+        )
+        tests = tests_affected(commit_range)
+        logger.info("Identified %s affected tests" % len(tests))
+
+        if not tests:
+            logger.info("Quitting because no tests were affected.")
+            return
+    else:
+        tests = []
+        logger.info("Running all tests")
+
+    wpt_args += [
+        "--log-tbpl=../artifacts/log_tbpl.log",
+        "--log-tbpl-level=info",
+        "--log-mach=-",
+        "-y",
+        "--no-pause",
+        "--no-restart-on-unexpected",
+        "--install-fonts"
+    ]
+    wpt_args += browser_specific_args.get(product, [])
+
+    command = ["python", "./wpt", "run"] + wpt_args + [product] + tests
+
+    logger.info("Executing command: %s" % " ".join(command))
+
+    subprocess.check_call(command)
+
+    wptreport = find_wptreport(wpt_args)
+    if wptreport:
+        gzip_file(wptreport)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=main.__doc__)
+    parser.add_argument("--commit-range", action="store",
+                        help="""Git commit range. If specified, this will be
+                             supplied to the `wpt tests-affected` command to
+                             determine the list of test to execute""")
+    parser.add_argument("product", action="store",
+                        help="Browser to run tests in")
+    parser.add_argument("wpt_args", nargs="*",
+                        help="Arguments to forward to `wpt run` command")
+    main(**vars(parser.parse_args()))