From 29b5fbdd8d2dd053ac503cb0fac56afdfd0c2b3d Mon Sep 17 00:00:00 2001 From: Mark Hammond Date: Thu, 3 Feb 2022 05:15:55 +0000 Subject: [PATCH] Bug 1752768 - upgrade Sync's TPS test suite to python3. r=LougeniaBailey Differential Revision: https://phabricator.services.mozilla.com/D137396 --- testing/tps/README | 28 ++++++---- testing/tps/create_venv.py | 92 +++++++++++++------------------- testing/tps/mach_commands.py | 2 +- testing/tps/setup.py | 18 +++---- testing/tps/tps/__init__.py | 1 - testing/tps/tps/cli.py | 2 - testing/tps/tps/firefoxrunner.py | 2 - testing/tps/tps/phase.py | 2 - testing/tps/tps/testrunner.py | 11 ++-- 9 files changed, 68 insertions(+), 90 deletions(-) diff --git a/testing/tps/README b/testing/tps/README index c9caf34b09ba..d1dd105860fb 100644 --- a/testing/tps/README +++ b/testing/tps/README @@ -7,23 +7,28 @@ Installation TPS requires several packages to operate properly. To install TPS and required packages, use the INSTALL.sh script, provided: - python create_venv.py /path/to/create/virtualenv + python3 create_venv.py /path/to/create/virtualenv -This script will create a virtalenv and install TPS into it. TPS can then -be run by activating the virtualenv and executing: +This script will create a virtalenv and install TPS into it. + +You must then activate the virtualenv by executing: + +- (linux): source /path/to/virtualenv/Scripts/activate +- (win): /path/to/virtualenv/Scripts/activate.bat + +TPS can then be run by executing: runtps --binary=/path/to/firefox +When you are done with TPS, you can deactivate the virtualenv by executing +`deactivate` Configuration ============= To edit the TPS configuration, do not edit config/config.json.in in the tree. -Instead, edit config.json inside your virtualenv; it will be located at -something like: - - (linux): /path/to/virtualenv/lib/python2.6/site-packages/tps-0.2.40-py2.6.egg/tps/config.json - (win): /path/to/virtualenv/Lib/site-packages/tps-0.2.40-py2.6.egg/tps/config.json - +Instead, edit config.json inside your virtualenv; it will be located at the +top level of where you specified the virtualenv be created - eg, for the +example above, it will be `/path/to/create/virtualenv/config.json` Setting Up Test Accounts ======================== @@ -39,4 +44,7 @@ following steps: 4. Go back to the Restmail URL, reload the page 5. Search for the verification link and open that page -Now you will be able to use your setup Firefox Account for Sync. +Now you will be able to use this account for TPS. Note that these +steps can be done in either a test profile or in a private browsing window - you +might want to avoid doing that in a "real" profile that's already connected to +Sync. diff --git a/testing/tps/create_venv.py b/testing/tps/create_venv.py index 0114dea1ff94..c64ae50b866b 100755 --- a/testing/tps/create_venv.py +++ b/testing/tps/create_venv.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. @@ -9,15 +9,11 @@ It's probably best to specify a path NOT inside the repo, otherwise all the virtualenv files will show up in e.g. hg status. """ -from __future__ import absolute_import, print_function - -import six import optparse import os -import shutil import subprocess import sys -import zipfile +import venv here = os.path.dirname(os.path.abspath(__file__)) @@ -38,58 +34,18 @@ See runtps --help for all options *********************************************************************** """ -# Link to the folder, which contains the zip archives of virtualenv -URL_VIRTUALENV = "https://codeload.github.com/pypa/virtualenv/zip/" -VERSION_VIRTUALENV = "15.0.0" - - if sys.platform == "win32": bin_name = os.path.join("Scripts", "activate.bat") - activate_env = os.path.join("Scripts", "activate_this.py") python_env = os.path.join("Scripts", "python.exe") else: bin_name = os.path.join("bin", "activate") - activate_env = os.path.join("bin", "activate_this.py") python_env = os.path.join("bin", "python") -def download(url, target): - """Downloads the specified url to the given target.""" - response = six.moves.urllib.request.urlopen(url) - with open(target, "wb") as f: - f.write(response.read()) - - return target - - -def setup_virtualenv(target, python_bin=None): - script_path = os.path.join( - here, "virtualenv-%s" % VERSION_VIRTUALENV, "virtualenv.py" - ) - - print("Downloading virtualenv {}".format(VERSION_VIRTUALENV)) - zip_path = download( - URL_VIRTUALENV + VERSION_VIRTUALENV, os.path.join(here, "virtualenv.zip") - ) - - try: - with zipfile.ZipFile(zip_path, "r") as f: - f.extractall(here) - - print("Creating new virtual environment") - cmd_args = [sys.executable, script_path, target] - - if python_bin: - cmd_args.extend(["-p", python_bin]) - - subprocess.check_call(cmd_args) - finally: - try: - os.remove(zip_path) - except OSError: - pass - - shutil.rmtree(os.path.dirname(script_path), ignore_errors=True) +def setup_virtualenv(target): + print("Creating new virtual environment:", os.path.abspath(target)) + # system_site_packages=True so we have access to setuptools. + venv.create(target, system_site_packages=True) def update_configfile(source, target, replacements): @@ -97,7 +53,7 @@ def update_configfile(source, target, replacements): with open(source) as config: for line in config: - for source_string, target_string in six.iteritems(replacements): + for source_string, target_string in replacements.items(): if target_string: line = line.replace(source_string, target_string) lines.append(line) @@ -172,11 +128,10 @@ def main(): target = args[0] assert target - setup_virtualenv(target, python_bin=options.python) + setup_virtualenv(target) # Activate tps environment - tps_env = os.path.join(target, activate_env) - exec(open(tps_env).read(), dict(__file__=tps_env)) + activate(target) # Install TPS in environment subprocess.check_call( @@ -217,5 +172,34 @@ def main(): print(usage_message.format(TARGET=target, BIN_NAME=bin_name)) +def activate(target): + # This is a lightly modified copy of `activate_this.py`, which existed when + # venv was an external package, but doesn't come with the builtin venv support. + old_os_path = os.environ.get("PATH", "") + os.environ["PATH"] = ( + os.path.dirname(os.path.abspath(__file__)) + os.pathsep + old_os_path + ) + base = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + if sys.platform == "win32": + site_packages = os.path.join(base, "Lib", "site-packages") + else: + site_packages = os.path.join( + base, "lib", "python%s" % sys.version[:3], "site-packages" + ) + prev_sys_path = list(sys.path) + import site + + site.addsitedir(site_packages) + sys.real_prefix = sys.prefix + sys.prefix = base + # Move the added items to the front of the path: + new_sys_path = [] + for item in list(sys.path): + if item not in prev_sys_path: + new_sys_path.append(item) + sys.path.remove(item) + sys.path[:0] = new_sys_path + + if __name__ == "__main__": main() diff --git a/testing/tps/mach_commands.py b/testing/tps/mach_commands.py index f313cd5315dd..34b20ff7996d 100644 --- a/testing/tps/mach_commands.py +++ b/testing/tps/mach_commands.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, # You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function + import os from mach.decorators import Command, CommandArgument diff --git a/testing/tps/setup.py b/testing/tps/setup.py index 9a9accfeb795..731eb4b9eaa5 100644 --- a/testing/tps/setup.py +++ b/testing/tps/setup.py @@ -2,30 +2,24 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import from setuptools import setup, find_packages -import sys version = "0.6" deps = [ "httplib2 == 0.9.2", "mozfile >= 1.2", - "mozhttpd == 0.7", - "mozinfo >= 0.10", - "mozinstall == 1.16", - "mozprocess == 0.26", + "wptserve >= 3.0", + "mozinfo >= 1.2", + "mozinstall == 2.0.1", + "mozprocess == 1.3", "mozprofile ~= 2.1", - "mozrunner ~= 7.2", - "mozversion == 1.5", + "mozrunner ~= 8.2", + "mozversion == 2.3", "PyYAML >= 4.0", ] -# we only support python 2.6+ right now -assert sys.version_info[0] == 2 -assert sys.version_info[1] >= 6 - setup( name="tps", version=version, diff --git a/testing/tps/tps/__init__.py b/testing/tps/tps/__init__.py index 115971f13e28..a4c384c1cea8 100644 --- a/testing/tps/tps/__init__.py +++ b/testing/tps/tps/__init__.py @@ -4,7 +4,6 @@ # flake8: noqa -from __future__ import absolute_import from .firefoxrunner import TPSFirefoxRunner from .testrunner import TPSTestRunner diff --git a/testing/tps/tps/cli.py b/testing/tps/tps/cli.py index e992b7a12f4b..9ef6186b4da1 100644 --- a/testing/tps/tps/cli.py +++ b/testing/tps/tps/cli.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import json import optparse import os diff --git a/testing/tps/tps/firefoxrunner.py b/testing/tps/tps/firefoxrunner.py index 7507b2ce2d06..0f2154916718 100644 --- a/testing/tps/tps/firefoxrunner.py +++ b/testing/tps/tps/firefoxrunner.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import httplib2 import os diff --git a/testing/tps/tps/phase.py b/testing/tps/tps/phase.py index 73a504445a57..f793d3083139 100644 --- a/testing/tps/tps/phase.py +++ b/testing/tps/tps/phase.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import - import re import os.path diff --git a/testing/tps/tps/testrunner.py b/testing/tps/tps/testrunner.py index 3c0b312cc5c9..3e7cf28025cd 100644 --- a/testing/tps/tps/testrunner.py +++ b/testing/tps/tps/testrunner.py @@ -2,7 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, division, print_function import json import yaml @@ -12,7 +11,7 @@ import tempfile import time import traceback -from mozhttpd import MozHttpd +from wptserve import server import mozinfo from mozprofile import Profile import mozversion @@ -341,7 +340,7 @@ class TPSTestRunner(object): tmplogfile = None if logdata: tmplogfile = TempFile(prefix="tps_log_") - tmplogfile.write(logdata) + tmplogfile.write(logdata.encode("utf-8")) tmplogfile.close() self.errorlogs[testname] = tmplogfile @@ -477,8 +476,8 @@ class TPSTestRunner(object): testlist = [os.path.basename(self.testfile)] testdir = os.path.dirname(self.testfile) - self.mozhttpd = MozHttpd(port=4567, docroot=testdir) - self.mozhttpd.start() + self.server = server.WebTestHttpd(port=4567, doc_root=testdir) + self.server.start() # run each test, and save the results for test in testlist: @@ -508,7 +507,7 @@ class TPSTestRunner(object): ) break - self.mozhttpd.stop() + self.server.stop() # generate the postdata we'll use to post the results to the db self.postdata = {