Run base wptrunner tests on python 3. (#16174)

tools/wptrunner/tox.ini

@@ -2,7 +2,8 @@
 xfail_strict=true
 
 [tox]
-envlist = py27-{base,chrome,edge,firefox,ie,opera,safari,sauce,servo}
+envlist = py27-{base,chrome,edge,firefox,ie,opera,safari,sauce,servo},py36-base
+skip_missing_interpreters = true
 
 [testenv]
 deps =

tools/wptrunner/wptrunner/tests/test_manifestexpected.py

@@ -1,10 +1,13 @@
+import sys
 from io import BytesIO
 
 import pytest
 
 from .. import manifestexpected
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="bytes/text confusion in py3")
 @pytest.mark.parametrize("fuzzy, expected", [
     (b"ref.html:1;200", [("ref.html", ((1, 1), (200, 200)))]),
     (b"ref.html:0-1;100-200", [("ref.html", ((0, 1), (100, 200)))]),

tools/wptrunner/wptrunner/tests/test_products.py

@@ -2,6 +2,7 @@
 
 import mock
 import pytest
+import sys
 
 from .base import all_products, active_products
 from .. import environment
@@ -18,7 +19,9 @@ def test_load_active_product(product):
     # test passes if it doesn't throw
 
 
-@all_products("product")
+@all_products("product", marks={
+    "firefox": pytest.mark.xfail(sys.version[0] == "3", reason="mozinfo 0.10 doesn't support py3"),
+})
 def test_load_all_products(product):
     """test every product either loads or throws ImportError"""
     try:

tools/wptrunner/wptrunner/tests/test_testloader.py

@@ -20,6 +20,8 @@
 
 @pytest.mark.xfail(sys.platform == "win32",
                    reason="NamedTemporaryFile cannot be reopened on Win32")
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="wptmanifest.parser doesn't support py3")
 def test_filter_unicode():
     tests = make_mock_manifest(("test", "a", 10), ("test", "a/b", 10),
                                ("test", "c", 10))

tools/wptrunner/wptrunner/tests/test_update.py

@@ -1,6 +1,7 @@
 import json
 import mock
 import os
+import pytest
 import sys
 from io import BytesIO
 
@@ -97,6 +98,8 @@ def create_test_manifest(tests, url_base="/"):
     return m
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="metadata doesn't support py3")
 def test_update_0():
     tests = [("path/to/test.htm", ["/path/to/test.htm"], "testharness",
               """[test.htm]
@@ -117,6 +120,8 @@ def test_update_0():
     assert updated[0][1].is_empty
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="metadata doesn't support py3")
 def test_update_1():
     test_id = "/path/to/test.htm"
     tests = [("path/to/test.htm", [test_id], "testharness",
@@ -139,6 +144,8 @@ def test_update_1():
     assert new_manifest.get_test(test_id).children[0].get("expected") == "FAIL"
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="metadata doesn't support py3")
 def test_skip_0():
     test_id = "/path/to/test.htm"
     tests = [("path/to/test.htm", [test_id], "testharness",
@@ -158,6 +165,8 @@ def test_skip_0():
     assert not updated
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="metadata doesn't support py3")
 def test_new_subtest():
     test_id = "/path/to/test.htm"
     tests = [("path/to/test.htm", [test_id], "testharness", """[test.htm]
@@ -182,6 +191,8 @@ def test_new_subtest():
     assert new_manifest.get_test(test_id).children[1].get("expected") == "FAIL"
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="metadata doesn't support py3")
 def test_update_multiple_0():
     test_id = "/path/to/test.htm"
     tests = [("path/to/test.htm", [test_id], "testharness", """[test.htm]
@@ -216,6 +227,8 @@ def test_update_multiple_0():
         "expected", {"debug": False, "os": "linux"}) == "TIMEOUT"
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="metadata doesn't support py3")
 def test_update_multiple_1():
     test_id = "/path/to/test.htm"
     tests = [("path/to/test.htm", [test_id], "testharness", """[test.htm]
@@ -252,6 +265,8 @@ def test_update_multiple_1():
         "expected", {"debug": False, "os": "windows"}) == "FAIL"
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="metadata doesn't support py3")
 def test_update_multiple_2():
     test_id = "/path/to/test.htm"
     tests = [("path/to/test.htm", [test_id], "testharness", """[test.htm]
@@ -286,6 +301,8 @@ def test_update_multiple_2():
         "expected", {"debug": True, "os": "osx"}) == "TIMEOUT"
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="metadata doesn't support py3")
 def test_update_multiple_3():
     test_id = "/path/to/test.htm"
     tests = [("path/to/test.htm", [test_id], "testharness", """[test.htm]
@@ -322,6 +339,8 @@ def test_update_multiple_3():
         "expected", {"debug": True, "os": "osx"}) == "TIMEOUT"
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="metadata doesn't support py3")
 def test_update_ignore_existing():
     test_id = "/path/to/test.htm"
     tests = [("path/to/test.htm", [test_id], "testharness", """[test.htm]
@@ -358,6 +377,8 @@ def test_update_ignore_existing():
         "expected", {"debug": False, "os": "osx"}) == "NOTRUN"
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="metadata doesn't support py3")
 def test_update_assertion_count_0():
     test_id = "/path/to/test.htm"
     tests = [("path/to/test.htm", [test_id], "testharness", """[test.htm]
@@ -381,6 +402,8 @@ def test_update_assertion_count_0():
     assert new_manifest.get_test(test_id).get("min-asserts") == 2
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="metadata doesn't support py3")
 def test_update_assertion_count_1():
     test_id = "/path/to/test.htm"
     tests = [("path/to/test.htm", [test_id], "testharness", """[test.htm]
@@ -404,6 +427,8 @@ def test_update_assertion_count_1():
     assert new_manifest.get_test(test_id).has_key("min-asserts") is False
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="metadata doesn't support py3")
 def test_update_assertion_count_2():
     test_id = "/path/to/test.htm"
     tests = [("path/to/test.htm", [test_id], "testharness", """[test.htm]
@@ -423,6 +448,8 @@ def test_update_assertion_count_2():
     assert not updated
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="metadata doesn't support py3")
 def test_update_assertion_count_3():
     test_id = "/path/to/test.htm"
     tests = [("path/to/test.htm", [test_id], "testharness", """[test.htm]
@@ -456,6 +483,8 @@ def test_update_assertion_count_3():
     assert new_manifest.get_test(test_id).get("min-asserts") == 2
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="metadata doesn't support py3")
 def test_update_assertion_count_4():
     test_id = "/path/to/test.htm"
     tests = [("path/to/test.htm", [test_id], "testharness", """[test.htm]""")]
@@ -486,6 +515,8 @@ def test_update_assertion_count_4():
     assert new_manifest.get_test(test_id).has_key("min-asserts") is False
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="metadata doesn't support py3")
 def test_update_lsan_0():
     test_id = "/path/to/test.htm"
     dir_id = "path/to/__dir__"
@@ -503,6 +534,8 @@ def test_update_lsan_0():
     assert new_manifest.get("lsan-allowed") == ["foo"]
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="metadata doesn't support py3")
 def test_update_lsan_1():
     test_id = "/path/to/test.htm"
     dir_id = "path/to/__dir__"
@@ -523,6 +556,8 @@ def test_update_lsan_1():
     assert new_manifest.get("lsan-allowed") == ["baz", "foo"]
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="metadata doesn't support py3")
 def test_update_lsan_2():
     test_id = "/path/to/test.htm"
     dir_id = "path/to/__dir__"
@@ -545,6 +580,8 @@ def test_update_lsan_2():
     assert new_manifest.get("lsan-allowed") == ["baz"]
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="metadata doesn't support py3")
 def test_update_lsan_3():
     test_id = "/path/to/test.htm"
     dir_id = "path/to/__dir__"
@@ -567,6 +604,8 @@ def test_update_lsan_3():
     assert new_manifest.get("lsan-allowed") == ["baz", "foo"]
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="metadata doesn't support py3")
 def test_update_wptreport_0():
     tests = [("path/to/test.htm", ["/path/to/test.htm"], "testharness",
               """[test.htm]
@@ -587,6 +626,8 @@ def test_update_wptreport_0():
     assert updated[0][1].is_empty
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="metadata doesn't support py3")
 def test_update_wptreport_1():
     tests = [("path/to/test.htm", ["/path/to/test.htm"], "testharness", ""),
              ("path/to/__dir__", ["path/to/__dir__"], None, "")]
@@ -602,6 +643,8 @@ def test_update_wptreport_1():
     assert updated[0][1].get("lsan-allowed") == ["baz"]
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="metadata doesn't support py3")
 def test_update_leak_total_0():
     test_id = "/path/to/test.htm"
     dir_id = "path/to/__dir__"
@@ -621,6 +664,8 @@ def test_update_leak_total_0():
     assert new_manifest.get("leak-threshold") == ['default:51200']
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="metadata doesn't support py3")
 def test_update_leak_total_1():
     test_id = "/path/to/test.htm"
     dir_id = "path/to/__dir__"
@@ -637,6 +682,8 @@ def test_update_leak_total_1():
     assert not updated
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="metadata doesn't support py3")
 def test_update_leak_total_2():
     test_id = "/path/to/test.htm"
     dir_id = "path/to/__dir__"
@@ -654,6 +701,8 @@ def test_update_leak_total_2():
     assert not updated
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="metadata doesn't support py3")
 def test_update_leak_total_3():
     test_id = "/path/to/test.htm"
     dir_id = "path/to/__dir__"
@@ -674,6 +723,8 @@ def test_update_leak_total_3():
     assert new_manifest.get("leak-threshold") == ['default:51200']
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="metadata doesn't support py3")
 def test_update_leak_total_4():
     test_id = "/path/to/test.htm"
     dir_id = "path/to/__dir__"
@@ -706,6 +757,8 @@ def create(self, state):
 class UpdateRunner(StepRunner):
     steps = [TestStep]
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="update.state doesn't support py3")
 def test_update_pickle():
     logger = structuredlog.StructuredLogger("expected_test")
     args = {

tools/wptrunner/wptrunner/tests/test_wpttest.py

@@ -1,5 +1,6 @@
+import pytest
+import sys
 from io import BytesIO
-
 from mock import Mock
 
 from manifest import manifest as wptmanifest
@@ -65,6 +66,8 @@ def make_mock_manifest(*items):
     return rv
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="bytes/text confusion in py3")
 def test_metadata_inherit():
     tests = make_mock_manifest(("test", "a", 10), ("test", "a/b", 10),
                                ("test", "c", 10))
@@ -89,6 +92,8 @@ def test_metadata_inherit():
     assert test_obj.tags == {"a", "dir:a"}
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="bytes/text confusion in py3")
 def test_conditional():
     tests = make_mock_manifest(("test", "a", 10), ("test", "a/b", 10),
                                ("test", "c", 10))
@@ -105,6 +110,8 @@ def test_conditional():
     assert test_obj.expected() == "FAIL"
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="bytes/text confusion in py3")
 def test_metadata_lsan_stack_depth():
     tests = make_mock_manifest(("test", "a", 10), ("test", "a/b", 10))
 
@@ -143,6 +150,8 @@ def test_metadata_lsan_stack_depth():
     assert test_obj.lsan_max_stack_depth == 42
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="bytes/text confusion in py3")
 def test_metadata_fuzzy():
     manifest_data = {
         "items": {"reftest": {"a/fuzzy.html": [["a/fuzzy.html",

tools/wptrunner/wptrunner/wptmanifest/tests/test_conditional.py

@@ -1,9 +1,13 @@
+import pytest
+import sys
 import unittest
 
 from ..backends import conditional
 from ..node import BinaryExpressionNode, BinaryOperatorNode, VariableNode, NumberNode
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="wptmanifest.parser doesn't support py3")
 class TestConditional(unittest.TestCase):
     def compile(self, input_text):
         return conditional.compile(input_text)

tools/wptrunner/wptrunner/wptmanifest/tests/test_parser.py

@@ -1,3 +1,5 @@
+import pytest
+import sys
 import unittest
 
 from six.moves import cStringIO as StringIO
@@ -8,6 +10,8 @@
 # use test_serializer for the majority of cases
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="wptmanifest.parser doesn't support py3")
 class TestExpression(unittest.TestCase):
     def setUp(self):
         self.parser = parser.Parser()

tools/wptrunner/wptrunner/wptmanifest/tests/test_serializer.py

@@ -6,6 +6,8 @@
 from .. import parser, serializer
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="wptmanifest.parser doesn't support py3")
 class TokenizerTest(unittest.TestCase):
     def setUp(self):
         self.serializer = serializer.ManifestSerializer()

tools/wptrunner/wptrunner/wptmanifest/tests/test_static.py

@@ -1,3 +1,5 @@
+import pytest
+import sys
 import unittest
 
 from ..backends import static
@@ -6,6 +8,8 @@
 # use test_serializer for the majority of cases
 
 
+@pytest.mark.xfail(sys.version[0] == "3",
+                   reason="wptmanifest.parser doesn't support py3")
 class TestStatic(unittest.TestCase):
     def compile(self, input_text, input_data):
         return static.compile(input_text, input_data)