From 0ad26e7e37dd3eb1300d7aa1ab1f5e426fedb576 Mon Sep 17 00:00:00 2001 From: Ms2ger Date: Wed, 26 Sep 2018 10:47:34 +0000 Subject: [PATCH] Bug 1492992 [wpt PR 11769] - Various test fixes for python3 support., a=testonly Automatic update from web-platform-testsVarious test fixes for python3 support. (#11769) -- wpt-commits: 5bd512cdc76f762662915a2d3d9e49b021f14ff6 wpt-pr: 11769 --- .../tools/wptserve/tests/functional/base.py | 2 +- .../tests/functional/test_handlers.py | 3 -- .../wptserve/tests/functional/test_pipes.py | 52 +++++++------------ .../wptserve/tests/functional/test_request.py | 3 +- .../tests/tools/wptserve/wptserve/pipes.py | 19 ++++--- .../tests/tools/wptserve/wptserve/request.py | 32 +++++++++--- .../tests/tools/wptserve/wptserve/response.py | 4 +- 7 files changed, 59 insertions(+), 56 deletions(-) diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/base.py b/testing/web-platform/tests/tools/wptserve/tests/functional/base.py index e49c8287c598..741ab0bcb745 100644 --- a/testing/web-platform/tests/tools/wptserve/tests/functional/base.py +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/base.py @@ -75,7 +75,7 @@ class TestUsingServer(unittest.TestCase): req.add_data(body) if auth is not None: - req.add_header("Authorization", "Basic %s" % base64.b64encode('%s:%s' % auth)) + req.add_header("Authorization", b"Basic %s" % base64.b64encode(("%s:%s" % auth).encode("utf-8"))) return urlopen(req) diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/test_handlers.py b/testing/web-platform/tests/tools/wptserve/tests/functional/test_handlers.py index 15c7cb7e803d..dc1518005156 100644 --- a/testing/web-platform/tests/tools/wptserve/tests/functional/test_handlers.py +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/test_handlers.py @@ -88,19 +88,16 @@ class TestFileHandler(TestUsingServer): self.request("/document.txt", headers={"Range":"bytes=%i-%i" % (len(expected), len(expected) + 10)}) self.assertEqual(cm.exception.code, 416) - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_config(self): resp = self.request("/sub.sub.txt") expected = b"localhost localhost %i" % self.server.port assert resp.read().rstrip() == expected - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_headers(self): resp = self.request("/sub_headers.sub.txt", headers={"X-Test": "PASS"}) expected = b"PASS" assert resp.read().rstrip() == expected - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_params(self): resp = self.request("/sub_params.sub.txt", query="test=PASS") expected = b"PASS" diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/test_pipes.py b/testing/web-platform/tests/tools/wptserve/tests/functional/test_pipes.py index 83b2c621641c..7739af5e26e1 100644 --- a/testing/web-platform/tests/tools/wptserve/tests/functional/test_pipes.py +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/test_pipes.py @@ -57,40 +57,35 @@ class TestSlice(TestUsingServer): self.assertEqual(resp.read(), expected[:10]) class TestSub(TestUsingServer): - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_config(self): resp = self.request("/sub.txt", query="pipe=sub") - expected = "localhost localhost %i" % self.server.port + expected = b"localhost localhost %i" % self.server.port self.assertEqual(resp.read().rstrip(), expected) @pytest.mark.xfail(sys.platform == "win32", reason="https://github.com/web-platform-tests/wpt/issues/12949") - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_file_hash(self): resp = self.request("/sub_file_hash.sub.txt") - expected = """ + expected = b""" md5: JmI1W8fMHfSfCarYOSxJcw== sha1: nqpWqEw4IW8NjD6R375gtrQvtTo= sha224: RqQ6fMmta6n9TuA/vgTZK2EqmidqnrwBAmQLRQ== sha256: G6Ljg1uPejQxqFmvFOcV/loqnjPTW5GSOePOfM/u0jw= sha384: lkXHChh1BXHN5nT5BYhi1x67E1CyYbPKRKoF2LTm5GivuEFpVVYtvEBHtPr74N9E -sha512: r8eLGRTc7ZznZkFjeVLyo6/FyQdra9qmlYCwKKxm3kfQAswRS9+3HsYk3thLUhcFmmWhK4dXaICz -JwGFonfXwg==""" +sha512: r8eLGRTc7ZznZkFjeVLyo6/FyQdra9qmlYCwKKxm3kfQAswRS9+3HsYk3thLUhcFmmWhK4dXaICzJwGFonfXwg==""" self.assertEqual(resp.read().rstrip(), expected.strip()) def test_sub_file_hash_unrecognized(self): with self.assertRaises(urllib.error.HTTPError): self.request("/sub_file_hash_unrecognized.sub.txt") - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_headers(self): resp = self.request("/sub_headers.txt", query="pipe=sub", headers={"X-Test": "PASS"}) - expected = "PASS" + expected = b"PASS" self.assertEqual(resp.read().rstrip(), expected) @pytest.mark.xfail(sys.platform == "win32", reason="https://github.com/web-platform-tests/wpt/issues/12949") - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_location(self): resp = self.request("/sub_location.sub.txt?query_string") expected = """ @@ -101,30 +96,27 @@ pathname: /sub_location.sub.txt port: {0} query: ?query_string scheme: http -server: http://localhost:{0}""".format(self.server.port) +server: http://localhost:{0}""".format(self.server.port).encode("ascii") self.assertEqual(resp.read().rstrip(), expected.strip()) - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_params(self): resp = self.request("/sub_params.txt", query="test=PASS&pipe=sub") - expected = "PASS" + expected = b"PASS" self.assertEqual(resp.read().rstrip(), expected) - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_url_base(self): resp = self.request("/sub_url_base.sub.txt") - self.assertEqual(resp.read().rstrip(), "Before / After") + self.assertEqual(resp.read().rstrip(), b"Before / After") - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_uuid(self): resp = self.request("/sub_uuid.sub.txt") - self.assertRegexpMatches(resp.read().rstrip(), r"Before [a-f0-9-]+ After") + self.assertRegexpMatches(resp.read().rstrip(), b"Before [a-f0-9-]+ After") - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_sub_var(self): resp = self.request("/sub_var.sub.txt") port = self.server.port - expected = "localhost %s A %s B localhost C" % (port, port) + print(port, type(port)) + expected = b"localhost %d A %d B localhost C" % (port, port) self.assertEqual(resp.read().rstrip(), expected) class TestTrickle(TestUsingServer): @@ -144,12 +136,10 @@ class TestTrickle(TestUsingServer): self.assertEqual(resp.info()["Expires"], "0") class TestPipesWithVariousHandlers(TestUsingServer): - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_with_python_file_handler(self): resp = self.request("/test_string.py", query="pipe=slice(null,2)") - self.assertEqual(resp.read(), "PA") + self.assertEqual(resp.read(), b"PA") - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_with_python_func_handler(self): @wptserve.handlers.handler def handler(request, response): @@ -157,9 +147,8 @@ class TestPipesWithVariousHandlers(TestUsingServer): route = ("GET", "/test/test_pipes_1/", handler) self.server.router.register(*route) resp = self.request(route[1], query="pipe=slice(null,2)") - self.assertEqual(resp.read(), "PA") + self.assertEqual(resp.read(), b"PA") - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_with_python_func_handler_using_response_writer(self): @wptserve.handlers.handler def handler(request, response): @@ -168,9 +157,8 @@ class TestPipesWithVariousHandlers(TestUsingServer): self.server.router.register(*route) resp = self.request(route[1], query="pipe=slice(null,2)") # slice has not been applied to the response, because response.writer was used. - self.assertEqual(resp.read(), "PASS") + self.assertEqual(resp.read(), b"PASS") - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_header_pipe_with_python_func_using_response_writer(self): @wptserve.handlers.handler def handler(request, response): @@ -180,9 +168,8 @@ class TestPipesWithVariousHandlers(TestUsingServer): resp = self.request(route[1], query="pipe=header(X-TEST,FAIL)") # header pipe was ignored, because response.writer was used. self.assertFalse(resp.info().get("X-TEST")) - self.assertEqual(resp.read(), "CONTENT") + self.assertEqual(resp.read(), b"CONTENT") - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_with_json_handler(self): @wptserve.handlers.json_handler def handler(request, response): @@ -190,32 +177,29 @@ class TestPipesWithVariousHandlers(TestUsingServer): route = ("GET", "/test/test_pipes_2/", handler) self.server.router.register(*route) resp = self.request(route[1], query="pipe=slice(null,2)") - self.assertEqual(resp.read(), '"{') + self.assertEqual(resp.read(), b'"{') - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_slice_with_as_is_handler(self): resp = self.request("/test.asis", query="pipe=slice(null,2)") self.assertEqual(202, resp.getcode()) self.assertEqual("Giraffe", resp.msg) self.assertEqual("PASS", resp.info()["X-Test"]) # slice has not been applied to the response, because response.writer was used. - self.assertEqual("Content", resp.read()) + self.assertEqual(b"Content", resp.read()) - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_headers_with_as_is_handler(self): resp = self.request("/test.asis", query="pipe=header(X-TEST,FAIL)") self.assertEqual(202, resp.getcode()) self.assertEqual("Giraffe", resp.msg) # header pipe was ignored. self.assertEqual("PASS", resp.info()["X-TEST"]) - self.assertEqual("Content", resp.read()) + self.assertEqual(b"Content", resp.read()) - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_trickle_with_as_is_handler(self): t0 = time.time() resp = self.request("/test.asis", query="pipe=trickle(1:d2:5:d1:r2)") t1 = time.time() - self.assertTrue('Content' in resp.read()) + self.assertTrue(b'Content' in resp.read()) self.assertGreater(6, t1-t0) if __name__ == '__main__': diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/test_request.py b/testing/web-platform/tests/tools/wptserve/tests/functional/test_request.py index 97d75eb71289..e5a0a7ad1292 100644 --- a/testing/web-platform/tests/tools/wptserve/tests/functional/test_request.py +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/test_request.py @@ -119,7 +119,6 @@ class TestRequest(TestUsingServer): class TestAuth(TestUsingServer): - @pytest.mark.xfail(sys.version_info >= (3,), reason="wptserve only works on Py2") def test_auth(self): @wptserve.handlers.handler def handler(request, response): @@ -129,4 +128,4 @@ class TestAuth(TestUsingServer): self.server.router.register(*route) resp = self.request(route[1], auth=("test", "PASS")) self.assertEqual(200, resp.getcode()) - self.assertEqual(["test", "PASS"], resp.read().split(" ")) + self.assertEqual([b"test", b"PASS"], resp.read().split(b" ")) diff --git a/testing/web-platform/tests/tools/wptserve/wptserve/pipes.py b/testing/web-platform/tests/tools/wptserve/wptserve/pipes.py index ad96f591a3a8..737b14d11cff 100644 --- a/testing/web-platform/tests/tools/wptserve/wptserve/pipes.py +++ b/testing/web-platform/tests/tools/wptserve/wptserve/pipes.py @@ -1,5 +1,6 @@ from cgi import escape from collections import deque +import base64 import gzip as gzip_module import hashlib import os @@ -273,8 +274,9 @@ def slice(request, response, start, end=None): (spelled "null" in a query string) to indicate the end of the file. """ - content = resolve_content(response) - response.content = content[start:end] + content = resolve_content(response)[start:end] + response.content = content + response.headers.set("Content-Length", len(content)) return response @@ -392,7 +394,7 @@ class SubFunctions(object): @staticmethod def file_hash(request, algorithm, path): - algorithm = algorithm.decode("ascii") + assert isinstance(algorithm, text_type) if algorithm not in SubFunctions.supported_algorithms: raise ValueError("Unsupported encryption algorithm: '%s'" % algorithm) @@ -400,7 +402,7 @@ class SubFunctions(object): absolute_path = os.path.join(request.doc_root, path) try: - with open(absolute_path) as f: + with open(absolute_path, "rb") as f: hash_obj.update(f.read()) except IOError: # In this context, an unhandled IOError will be interpreted by the @@ -410,7 +412,7 @@ class SubFunctions(object): # the path to the file to be hashed is invalid. raise Exception('Cannot open file for hash computation: "%s"' % absolute_path) - return hash_obj.digest().encode('base64').strip() + return base64.b64encode(hash_obj.digest()).strip() def template(request, content, escape_type="html"): #TODO: There basically isn't any error handling here @@ -425,7 +427,6 @@ def template(request, content, escape_type="html"): tokens = deque(tokens) token_type, field = tokens.popleft() - field = field.decode("ascii") if token_type == "var": variable = field @@ -490,7 +491,11 @@ def template(request, content, escape_type="html"): #Should possibly support escaping for other contexts e.g. script #TODO: read the encoding of the response - return escape_func(text_type(value)).encode("utf-8") + if isinstance(value, binary_type): + value = value.decode("utf-8") + elif isinstance(value, int): + value = text_type(value) + return escape_func(value).encode("utf-8") template_regexp = re.compile(br"{{([^}]*)}}") new_content = template_regexp.sub(config_replacement, content) diff --git a/testing/web-platform/tests/tools/wptserve/wptserve/request.py b/testing/web-platform/tests/tools/wptserve/wptserve/request.py index cb575ccdba45..233ff151c28e 100644 --- a/testing/web-platform/tests/tools/wptserve/wptserve/request.py +++ b/testing/web-platform/tests/tools/wptserve/wptserve/request.py @@ -1,7 +1,7 @@ import base64 import cgi from six.moves.http_cookies import BaseCookie -from six import BytesIO +from six import BytesIO, binary_type, text_type import tempfile from six.moves.urllib.parse import parse_qsl, urlsplit @@ -318,8 +318,8 @@ class Request(object): def cookies(self): if self._cookies is None: parser = BaseCookie() - cookie_headers = self.headers.get("cookie", "") - parser.load(cookie_headers) + cookie_headers = self.headers.get("cookie", u"") + parser.load(cookie_headers.encode("utf-8")) cookies = Cookies() for key, value in parser.iteritems(): cookies[key] = CookieValue(value) @@ -355,6 +355,16 @@ class H2Request(Request): super(H2Request, self).__init__(request_handler) +def maybedecode(s): + if isinstance(s, text_type): + return s + + if isinstance(s, binary_type): + return s.decode("ascii") + + raise TypeError("Unexpected value in RequestHeaders: %r" % s) + + class RequestHeaders(dict): """Dictionary-like API for accessing request headers.""" def __init__(self, items): @@ -369,15 +379,17 @@ class RequestHeaders(dict): for value in values: # getallmatchingheaders returns raw header lines, so # split to get name, value - multiples.append(value.split(':', 1)[1].strip()) - dict.__setitem__(self, key, multiples) + multiples.append(maybedecode(value).split(':', 1)[1].strip()) + headers = multiples else: - dict.__setitem__(self, key, [items[header]]) + headers = [maybedecode(items[header])] + dict.__setitem__(self, maybedecode(key), headers) def __getitem__(self, key): """Get all headers of a certain (case-insensitive) name. If there is more than one, the values are returned comma separated""" + key = maybedecode(key) values = dict.__getitem__(self, key.lower()) if len(values) == 1: return values[0] @@ -403,6 +415,7 @@ class RequestHeaders(dict): def get_list(self, key, default=missing): """Get all the header values for a particular field name as a list""" + key = maybedecode(key) try: return dict.__getitem__(self, key.lower()) except KeyError: @@ -412,6 +425,7 @@ class RequestHeaders(dict): raise def __contains__(self, key): + key = maybedecode(key) return dict.__contains__(self, key.lower()) def iteritems(self): @@ -599,6 +613,7 @@ class Authentication(object): if "authorization" in headers: header = headers.get("authorization") + assert isinstance(header, text_type) auth_type, data = header.split(" ", 1) if auth_type in auth_schemes: self.username, self.password = auth_schemes[auth_type](data) @@ -606,5 +621,6 @@ class Authentication(object): raise HTTPException(400, "Unsupported authentication scheme %s" % auth_type) def decode_basic(self, data): - decoded_data = base64.decodestring(data) - return decoded_data.split(":", 1) + assert isinstance(data, text_type) + decoded_data = base64.decodestring(data.encode("utf-8")) + return decoded_data.decode("utf-8").split(":", 1) diff --git a/testing/web-platform/tests/tools/wptserve/wptserve/response.py b/testing/web-platform/tests/tools/wptserve/wptserve/response.py index 44299cc994ef..483265bf0846 100644 --- a/testing/web-platform/tests/tools/wptserve/wptserve/response.py +++ b/testing/web-platform/tests/tools/wptserve/wptserve/response.py @@ -183,8 +183,10 @@ class Response(object): True, the entire content of the file will be returned as a string facilitating non-streaming operations like template substitution. """ - if isinstance(self.content, (binary_type, text_type)): + if isinstance(self.content, binary_type): yield self.content + elif isinstance(self.content, text_type): + yield self.content.encode("utf-8") elif hasattr(self.content, "read"): if read_file: yield self.content.read()