Skip to content

Commit

Permalink
Fix stability check when tests are skipped
Browse files Browse the repository at this point in the history
In #11570, we added a
check to reject slow tests in stability checks. The code didn't consider
an edge case where tests are skipped and don't contain "extra" info.

This change silently skips a test in this check if it does not have
necessary extra information.
  • Loading branch information
Hexcles committed Sep 6, 2018
1 parent cf0ea11 commit 332168f
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 2 deletions.
10 changes: 8 additions & 2 deletions tools/wptrunner/wptrunner/stability.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,8 +96,12 @@ def test_end(self, data):
duration = data["time"] - test.pop("start_time")
test["longest_duration"][data["status"]] = max(
duration, test["longest_duration"][data["status"]])
# test_timeout is in seconds; convert it to ms.
test["timeout"] = data["extra"]["test_timeout"] * 1000
try:
# test_timeout is in seconds; convert it to ms.
test["timeout"] = data["extra"]["test_timeout"] * 1000
except KeyError:
# If a test is skipped, it won't have extra info.
pass


def is_inconsistent(results_dict, iterations):
Expand All @@ -118,6 +122,8 @@ def find_slow_status(test):
A result status produced by a run that almost times out; None, if no
runs almost time out.
"""
if "timeout" not in test:
return None
threshold = test["timeout"] * FLAKY_THRESHOLD
for status in ['PASS', 'FAIL', 'OK', 'ERROR']:
if (status in test["longest_duration"] and
Expand Down
2 changes: 2 additions & 0 deletions tools/wptrunner/wptrunner/tests/test_stability.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,3 +29,5 @@ def test_find_slow_status():
assert stability.find_slow_status({
"longest_duration": {"TIMEOUT": 10, "FAIL": 81},
"timeout": 100}) == "FAIL"
assert stability.find_slow_status({
"longest_duration": {"SKIP": 0}}) is None

0 comments on commit 332168f

Please sign in to comment.