Various test fixes for python3 support.

tools/wptserve/tests/functional/base.py

@@ -75,7 +75,7 @@ def request(self, path, query=None, method="GET", headers=None, body=None, auth=
             req.add_data(body)
 
         if auth is not None:
-            req.add_header("Authorization", "Basic %s" % base64.b64encode('%s:%s' % auth))
+            req.add_header("Authorization", b"Basic %s" % base64.b64encode(("%s:%s" % auth).encode("utf-8")))
 
         return urlopen(req)
 

tools/wptserve/tests/functional/test_handlers.py

@@ -88,19 +88,16 @@ def test_range_invalid(self):
             self.request("/document.txt", headers={"Range":"bytes=%i-%i" % (len(expected), len(expected) + 10)})
         self.assertEqual(cm.exception.code, 416)
 
-    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_sub_config(self):
         resp = self.request("/sub.sub.txt")
         expected = b"localhost localhost %i" % self.server.port
         assert resp.read().rstrip() == expected
 
-    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_sub_headers(self):
         resp = self.request("/sub_headers.sub.txt", headers={"X-Test": "PASS"})
         expected = b"PASS"
         assert resp.read().rstrip() == expected
 
-    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_sub_params(self):
         resp = self.request("/sub_params.sub.txt", query="test=PASS")
         expected = b"PASS"

tools/wptserve/tests/functional/test_pipes.py

@@ -57,40 +57,35 @@ def test_no_lower(self):
         self.assertEqual(resp.read(), expected[:10])
 
 class TestSub(TestUsingServer):
-    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_sub_config(self):
         resp = self.request("/sub.txt", query="pipe=sub")
-        expected = "localhost localhost %i" % self.server.port
+        expected = b"localhost localhost %i" % self.server.port
         self.assertEqual(resp.read().rstrip(), expected)
 
     @pytest.mark.xfail(sys.platform == "win32",
                        reason="https://github.com/web-platform-tests/wpt/issues/12949")
-    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_sub_file_hash(self):
         resp = self.request("/sub_file_hash.sub.txt")
-        expected = """
+        expected = b"""
 md5: JmI1W8fMHfSfCarYOSxJcw==
 sha1: nqpWqEw4IW8NjD6R375gtrQvtTo=
 sha224: RqQ6fMmta6n9TuA/vgTZK2EqmidqnrwBAmQLRQ==
 sha256: G6Ljg1uPejQxqFmvFOcV/loqnjPTW5GSOePOfM/u0jw=
 sha384: lkXHChh1BXHN5nT5BYhi1x67E1CyYbPKRKoF2LTm5GivuEFpVVYtvEBHtPr74N9E
-sha512: r8eLGRTc7ZznZkFjeVLyo6/FyQdra9qmlYCwKKxm3kfQAswRS9+3HsYk3thLUhcFmmWhK4dXaICz
-JwGFonfXwg=="""
+sha512: r8eLGRTc7ZznZkFjeVLyo6/FyQdra9qmlYCwKKxm3kfQAswRS9+3HsYk3thLUhcFmmWhK4dXaICzJwGFonfXwg=="""
         self.assertEqual(resp.read().rstrip(), expected.strip())
 
     def test_sub_file_hash_unrecognized(self):
         with self.assertRaises(urllib.error.HTTPError):
             self.request("/sub_file_hash_unrecognized.sub.txt")
 
-    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_sub_headers(self):
         resp = self.request("/sub_headers.txt", query="pipe=sub", headers={"X-Test": "PASS"})
-        expected = "PASS"
+        expected = b"PASS"
         self.assertEqual(resp.read().rstrip(), expected)
 
     @pytest.mark.xfail(sys.platform == "win32",
                        reason="https://github.com/web-platform-tests/wpt/issues/12949")
-    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_sub_location(self):
         resp = self.request("/sub_location.sub.txt?query_string")
         expected = """
@@ -101,30 +96,27 @@ def test_sub_location(self):
 port: {0}
 query: ?query_string
 scheme: http
-server: http://localhost:{0}""".format(self.server.port)
+server: http://localhost:{0}""".format(self.server.port).encode("ascii")
         self.assertEqual(resp.read().rstrip(), expected.strip())
 
-    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_sub_params(self):
         resp = self.request("/sub_params.txt", query="test=PASS&pipe=sub")
-        expected = "PASS"
+        expected = b"PASS"
         self.assertEqual(resp.read().rstrip(), expected)
 
-    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_sub_url_base(self):
         resp = self.request("/sub_url_base.sub.txt")
-        self.assertEqual(resp.read().rstrip(), "Before / After")
+        self.assertEqual(resp.read().rstrip(), b"Before / After")
 
-    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_sub_uuid(self):
         resp = self.request("/sub_uuid.sub.txt")
-        self.assertRegexpMatches(resp.read().rstrip(), r"Before [a-f0-9-]+ After")
+        self.assertRegexpMatches(resp.read().rstrip(), b"Before [a-f0-9-]+ After")
 
-    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_sub_var(self):
         resp = self.request("/sub_var.sub.txt")
         port = self.server.port
-        expected = "localhost %s A %s B localhost C" % (port, port)
+        print(port, type(port))
+        expected = b"localhost %d A %d B localhost C" % (port, port)
         self.assertEqual(resp.read().rstrip(), expected)
 
 class TestTrickle(TestUsingServer):
@@ -144,22 +136,20 @@ def test_headers(self):
         self.assertEqual(resp.info()["Expires"], "0")
 
 class TestPipesWithVariousHandlers(TestUsingServer):
-    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_with_python_file_handler(self):
         resp = self.request("/test_string.py", query="pipe=slice(null,2)")
-        self.assertEqual(resp.read(), "PA")
+        self.assertEqual(resp.read(), b"PA")
 
-    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_with_python_func_handler(self):
         @wptserve.handlers.handler
         def handler(request, response):
             return "PASS"
         route = ("GET", "/test/test_pipes_1/", handler)
         self.server.router.register(*route)
         resp = self.request(route[1], query="pipe=slice(null,2)")
-        self.assertEqual(resp.read(), "PA")
+        self.assertEqual(resp.read(), b"PA")
 
-    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
+    @pytest.mark.xfail(sys.version_info >= (3,), reason="python3 is stricter")
     def test_with_python_func_handler_using_response_writer(self):
         @wptserve.handlers.handler
         def handler(request, response):
@@ -168,9 +158,9 @@ def handler(request, response):
         self.server.router.register(*route)
         resp = self.request(route[1], query="pipe=slice(null,2)")
         # slice has not been applied to the response, because response.writer was used.
-        self.assertEqual(resp.read(), "PASS")
+        self.assertEqual(resp.read(), b"PASS")
 
-    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
+    @pytest.mark.xfail(sys.version_info >= (3,), reason="python3 is stricter")
     def test_header_pipe_with_python_func_using_response_writer(self):
         @wptserve.handlers.handler
         def handler(request, response):
@@ -180,42 +170,38 @@ def handler(request, response):
         resp = self.request(route[1], query="pipe=header(X-TEST,FAIL)")
         # header pipe was ignored, because response.writer was used.
         self.assertFalse(resp.info().get("X-TEST"))
-        self.assertEqual(resp.read(), "CONTENT")
+        self.assertEqual(resp.read(), b"CONTENT")
 
-    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_with_json_handler(self):
         @wptserve.handlers.json_handler
         def handler(request, response):
             return json.dumps({'data': 'PASS'})
         route = ("GET", "/test/test_pipes_2/", handler)
         self.server.router.register(*route)
         resp = self.request(route[1], query="pipe=slice(null,2)")
-        self.assertEqual(resp.read(), '"{')
+        self.assertEqual(resp.read(), b'"{')
 
-    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_slice_with_as_is_handler(self):
         resp = self.request("/test.asis", query="pipe=slice(null,2)")
         self.assertEqual(202, resp.getcode())
         self.assertEqual("Giraffe", resp.msg)
         self.assertEqual("PASS", resp.info()["X-Test"])
         # slice has not been applied to the response, because response.writer was used.
-        self.assertEqual("Content", resp.read())
+        self.assertEqual(b"Content", resp.read())
 
-    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_headers_with_as_is_handler(self):
         resp = self.request("/test.asis", query="pipe=header(X-TEST,FAIL)")
         self.assertEqual(202, resp.getcode())
         self.assertEqual("Giraffe", resp.msg)
         # header pipe was ignored.
         self.assertEqual("PASS", resp.info()["X-TEST"])
-        self.assertEqual("Content", resp.read())
+        self.assertEqual(b"Content", resp.read())
 
-    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_trickle_with_as_is_handler(self):
         t0 = time.time()
         resp = self.request("/test.asis", query="pipe=trickle(1:d2:5:d1:r2)")
         t1 = time.time()
-        self.assertTrue('Content' in resp.read())
+        self.assertTrue(b'Content' in resp.read())
         self.assertGreater(6, t1-t0)
 
 if __name__ == '__main__':

tools/wptserve/tests/functional/test_request.py

@@ -1,5 +1,3 @@
-import sys
-
 import pytest
 
 wptserve = pytest.importorskip("wptserve")
@@ -119,7 +117,6 @@ def handler(request, response):
 
 
 class TestAuth(TestUsingServer):
-    @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2")
     def test_auth(self):
         @wptserve.handlers.handler
         def handler(request, response):
@@ -129,4 +126,4 @@ def handler(request, response):
         self.server.router.register(*route)
         resp = self.request(route[1], auth=("test", "PASS"))
         self.assertEqual(200, resp.getcode())
-        self.assertEqual(["test", "PASS"], resp.read().split(" "))
+        self.assertEqual([b"test", b"PASS"], resp.read().split(b" "))

tools/wptserve/wptserve/pipes.py

@@ -1,5 +1,6 @@
 from cgi import escape
 from collections import deque
+import base64
 import gzip as gzip_module
 import hashlib
 import os
@@ -273,8 +274,9 @@ def slice(request, response, start, end=None):
                 (spelled "null" in a query string) to indicate the end of
                 the file.
     """
-    content = resolve_content(response)
-    response.content = content[start:end]
+    content = resolve_content(response)[start:end]
+    response.content = content
+    response.headers.set("Content-Length", len(content))
     return response
 
 
@@ -392,15 +394,15 @@ def uuid(request):
 
     @staticmethod
     def file_hash(request, algorithm, path):
-        algorithm = algorithm.decode("ascii")
+        assert isinstance(algorithm, text_type)
         if algorithm not in SubFunctions.supported_algorithms:
             raise ValueError("Unsupported encryption algorithm: '%s'" % algorithm)
 
         hash_obj = getattr(hashlib, algorithm)()
         absolute_path = os.path.join(request.doc_root, path)
 
         try:
-            with open(absolute_path) as f:
+            with open(absolute_path, "rb") as f:
                 hash_obj.update(f.read())
         except IOError:
             # In this context, an unhandled IOError will be interpreted by the
@@ -410,7 +412,7 @@ def file_hash(request, algorithm, path):
             # the path to the file to be hashed is invalid.
             raise Exception('Cannot open file for hash computation: "%s"' % absolute_path)
 
-        return hash_obj.digest().encode('base64').strip()
+        return base64.b64encode(hash_obj.digest()).strip()
 
 def template(request, content, escape_type="html"):
     #TODO: There basically isn't any error handling here
@@ -425,7 +427,6 @@ def config_replacement(match):
         tokens = deque(tokens)
 
         token_type, field = tokens.popleft()
-        field = field.decode("ascii")
 
         if token_type == "var":
             variable = field
@@ -490,7 +491,11 @@ def config_replacement(match):
 
         #Should possibly support escaping for other contexts e.g. script
         #TODO: read the encoding of the response
-        return escape_func(text_type(value)).encode("utf-8")
+        if isinstance(value, binary_type):
+            value = value.decode("utf-8")
+        elif isinstance(value, int):
+            value = text_type(value)
+        return escape_func(value).encode("utf-8")
 
     template_regexp = re.compile(br"{{([^}]*)}}")
     new_content = template_regexp.sub(config_replacement, content)

tools/wptserve/wptserve/request.py

@@ -1,7 +1,7 @@
 import base64
 import cgi
 from six.moves.http_cookies import BaseCookie
-from six import BytesIO
+from six import BytesIO, binary_type, text_type
 import tempfile
 
 from six.moves.urllib.parse import parse_qsl, urlsplit
@@ -318,8 +318,8 @@ def POST(self):
     def cookies(self):
         if self._cookies is None:
             parser = BaseCookie()
-            cookie_headers = self.headers.get("cookie", "")
-            parser.load(cookie_headers)
+            cookie_headers = self.headers.get("cookie", u"")
+            parser.load(cookie_headers.encode("utf-8"))
             cookies = Cookies()
             for key, value in parser.iteritems():
                 cookies[key] = CookieValue(value)
@@ -355,6 +355,16 @@ def __init__(self, request_handler):
         super(H2Request, self).__init__(request_handler)
 
 
+def maybedecode(s):
+    if isinstance(s, text_type):
+        return s
+
+    if isinstance(s, binary_type):
+        return s.decode("ascii")
+
+    raise TypeError("Unexpected value in RequestHeaders: %r" % s)
+
+
 class RequestHeaders(dict):
     """Dictionary-like API for accessing request headers."""
     def __init__(self, items):
@@ -369,15 +379,18 @@ def __init__(self, items):
                 for value in values:
                     # getallmatchingheaders returns raw header lines, so
                     # split to get name, value
-                    multiples.append(value.split(':', 1)[1].strip())
-                dict.__setitem__(self, key, multiples)
+                    multiples.append(maybedecode(value).split(':', 1)[1].strip())
+                headers = multiples
             else:
-                dict.__setitem__(self, key, [items[header]])
+                headers = [maybedecode(items[header])]
+            print(key, headers, type(key), list(map(type, headers)))
+            dict.__setitem__(self, maybedecode(key), headers)
 
 
     def __getitem__(self, key):
         """Get all headers of a certain (case-insensitive) name. If there is
         more than one, the values are returned comma separated"""
+        key = maybedecode(key)
         values = dict.__getitem__(self, key.lower())
         if len(values) == 1:
             return values[0]
@@ -403,6 +416,7 @@ def get(self, key, default=None):
     def get_list(self, key, default=missing):
         """Get all the header values for a particular field name as
         a list"""
+        key = maybedecode(key)
         try:
             return dict.__getitem__(self, key.lower())
         except KeyError:
@@ -412,6 +426,7 @@ def get_list(self, key, default=missing):
                 raise
 
     def __contains__(self, key):
+        key = maybedecode(key)
         return dict.__contains__(self, key.lower())
 
     def iteritems(self):
@@ -599,12 +614,14 @@ def __init__(self, headers):
 
         if "authorization" in headers:
             header = headers.get("authorization")
+            assert isinstance(header, text_type)
             auth_type, data = header.split(" ", 1)
             if auth_type in auth_schemes:
                 self.username, self.password = auth_schemes[auth_type](data)
             else:
                 raise HTTPException(400, "Unsupported authentication scheme %s" % auth_type)
 
     def decode_basic(self, data):
-        decoded_data = base64.decodestring(data)
-        return decoded_data.split(":", 1)
+        assert isinstance(data, text_type)
+        decoded_data = base64.decodestring(data.encode("utf-8"))
+        return decoded_data.decode("utf-8").split(":", 1)

tools/wptserve/wptserve/response.py

@@ -183,8 +183,10 @@ def iter_content(self, read_file=False):
         True, the entire content of the file will be returned as a string facilitating
         non-streaming operations like template substitution.
         """
-        if isinstance(self.content, (binary_type, text_type)):
+        if isinstance(self.content, binary_type):
             yield self.content
+        elif isinstance(self.content, text_type):
+            yield self.content.encode("utf-8")
         elif hasattr(self.content, "read"):
             if read_file:
                 yield self.content.read()