From 3a2583759ba39cd8bc35e720c545275255517ada Mon Sep 17 00:00:00 2001 From: Dave Hunt Date: Fri, 8 Jun 2018 13:24:27 +0100 Subject: [PATCH] Bug 1466211 - Switch all |mach python-test| tests to run using pipenv; r=ahal MozReview-Commit-ID: AzmdDgAgZgI --HG-- extra : rebase_source : cd4603b707bd4ca3f7e0783cf5b9adb8c47a9e01 extra : source : e550be515be30148c0a89c33953dff4148a6a171 --- Pipfile | 4 - Pipfile.lock | 6 +- python/Pipfile | 36 --------- python/Pipfile.lock | 106 ------------------------- python/mach_commands.py | 7 +- python/mozbuild/mozbuild/base.py | 14 ++-- python/mozbuild/mozbuild/pipenv.txt | 5 -- python/mozbuild/mozbuild/virtualenv.py | 66 +++++++++++---- 8 files changed, 60 insertions(+), 184 deletions(-) delete mode 100644 python/Pipfile delete mode 100644 python/Pipfile.lock delete mode 100644 python/mozbuild/mozbuild/pipenv.txt diff --git a/Pipfile b/Pipfile index 01f2a3f6c36d..ff0a11ff98d6 100644 --- a/Pipfile +++ b/Pipfile @@ -16,7 +16,3 @@ requests = "==2.9.1" six = "==1.10.0" virtualenv = "==15.2.0" voluptuous = "==0.10.5" - - -[requires] -python_version = "2.7" diff --git a/Pipfile.lock b/Pipfile.lock index 177db7a767d9..da173523d990 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,12 +1,10 @@ { "_meta": { "hash": { - "sha256": "af4e239c88ce3d74e2e3dd7d352c3e8a203ce476c7369b2a4dc0eea7114996ba" + "sha256": "eb8b0a9771d4420f83fbbabf9952dc783aeefe9be455559de2f3ebff27caa93f" }, "pipfile-spec": 6, - "requires": { - "python_version": "2.7" - }, + "requires": {}, "sources": [ { "name": "pypi", diff --git a/python/Pipfile b/python/Pipfile deleted file mode 100644 index 5fc35cde698b..000000000000 --- a/python/Pipfile +++ /dev/null @@ -1,36 +0,0 @@ -[[source]] -url = "https://pypi.org/simple" -verify_ssl = true -name = "pypi" - -[packages] -"d5b4a14" = {path = "./mach"} -"8ddb376" = {path = "./mozbuild"} -"b3ddbcf" = {path = "./mozterm"} -"38a4a9a" = {path = "./mozversioncontrol"} -"26d92fb" = {path = "./../config/mozunit"} -"cea2946" = {path = "./../testing/mozbase/manifestparser"} -"ffcf6e6" = {path = "./../testing/mozbase/mozcrash"} -"195ae2e" = {path = "./../testing/mozbase/mozdebug"} -"8dab59a" = {path = "./../testing/mozbase/mozdevice"} -"58d0848" = {path = "./../testing/mozbase/mozfile"} -"fd0b608" = {path = "./../testing/mozbase/mozhttpd"} -"7329809" = {path = "./../testing/mozbase/mozinfo"} -"501835d" = {path = "./../testing/mozbase/mozinstall"} -"807c1c5" = {path = "./../testing/mozbase/mozlog"} -"e09e103" = {path = "./../testing/mozbase/moznetwork"} -"132adec" = {path = "./../testing/mozbase/mozprocess"} -"d88f467" = {path = "./../testing/mozbase/mozprofile"} -"1de94f2" = {path = "./../testing/mozbase/mozrunner"} -"6477f20" = {path = "./../testing/mozbase/moztest"} -"f1d74ca" = {path = "./../testing/mozbase/mozversion"} -"47200d8" = {path = "./../third_party/python/futures", markers="python_version < '3'"} -"110bcc4" = {path = "./../third_party/python/jsmin"} -"c49d32a" = {path = "./../third_party/python/mock-1.0.0", markers="python_version < '3.3'"} -"c2c21d9" = {path = "./../third_party/python/py"} -"f4b00e9" = {path = "./../third_party/python/pytest"} -"053111f" = {path = "./../third_party/python/requests"} -"d250320" = {path = "./../third_party/python/six"} -"f1de77a" = {path = "./../third_party/python/which", markers="python_version < '3.3'"} - -[dev-packages] diff --git a/python/Pipfile.lock b/python/Pipfile.lock deleted file mode 100644 index ede8984d4a66..000000000000 --- a/python/Pipfile.lock +++ /dev/null @@ -1,106 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "dfc219f64edc7715acdb35e03dcee665ec26908c18a58d3a3a88dda3ab393b17" - }, - "pipfile-spec": 6, - "requires": {}, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true - } - ] - }, - "default": { - "053111f": { - "path": "./../third_party/python/requests" - }, - "110bcc4": { - "path": "./../third_party/python/jsmin" - }, - "132adec": { - "path": "./../testing/mozbase/mozprocess" - }, - "195ae2e": { - "path": "./../testing/mozbase/mozdebug" - }, - "1de94f2": { - "path": "./../testing/mozbase/mozrunner" - }, - "26d92fb": { - "path": "./../config/mozunit" - }, - "38a4a9a": { - "path": "./mozversioncontrol" - }, - "47200d8": { - "markers": "python_version < '3'", - "path": "./../third_party/python/futures" - }, - "501835d": { - "path": "./../testing/mozbase/mozinstall" - }, - "58d0848": { - "path": "./../testing/mozbase/mozfile" - }, - "6477f20": { - "path": "./../testing/mozbase/moztest" - }, - "7329809": { - "path": "./../testing/mozbase/mozinfo" - }, - "807c1c5": { - "path": "./../testing/mozbase/mozlog" - }, - "8dab59a": { - "path": "./../testing/mozbase/mozdevice" - }, - "8ddb376": { - "path": "./mozbuild" - }, - "b3ddbcf": { - "path": "./mozterm" - }, - "c2c21d9": { - "path": "./../third_party/python/py" - }, - "c49d32a": { - "markers": "python_version < '3.3'", - "path": "./../third_party/python/mock-1.0.0" - }, - "cea2946": { - "path": "./../testing/mozbase/manifestparser" - }, - "d250320": { - "path": "./../third_party/python/six" - }, - "d5b4a14": { - "path": "./mach" - }, - "d88f467": { - "path": "./../testing/mozbase/mozprofile" - }, - "e09e103": { - "path": "./../testing/mozbase/moznetwork" - }, - "f1d74ca": { - "path": "./../testing/mozbase/mozversion" - }, - "f1de77a": { - "markers": "python_version < '3.3'", - "path": "./../third_party/python/which" - }, - "f4b00e9": { - "path": "./../third_party/python/pytest" - }, - "fd0b608": { - "path": "./../testing/mozbase/mozhttpd" - }, - "ffcf6e6": { - "path": "./../testing/mozbase/mozcrash" - } - }, - "develop": {} -} diff --git a/python/mach_commands.py b/python/mach_commands.py index f6429b1ef66b..b4a1e27d3c4b 100644 --- a/python/mach_commands.py +++ b/python/mach_commands.py @@ -99,11 +99,8 @@ class MachCommands(MachCommandBase): jobs=1, three=False, **kwargs): - if three: - # use pipenv to run tests against Python 3 - self.activate_pipenv(os.path.join(here, 'Pipfile'), ['--three']) - else: - self._activate_virtualenv() + pipenv_args = ['--three' if three else '--two'] + self.activate_pipenv(pipfile=None, args=pipenv_args, populate=True) if test_objects is None: from moztest.resolve import TestResolver diff --git a/python/mozbuild/mozbuild/base.py b/python/mozbuild/mozbuild/base.py index 6d8d756e2de2..f840bdf8eb75 100644 --- a/python/mozbuild/mozbuild/base.py +++ b/python/mozbuild/mozbuild/base.py @@ -752,16 +752,16 @@ class MozbuildObject(ProcessExecutionMixin): self._activate_virtualenv() pipenv = os.path.join(self.virtualenv_manager.bin_path, 'pipenv') if not os.path.exists(pipenv): - pipenv_reqs = os.path.join(self.topsrcdir, 'python/mozbuild/mozbuild/pipenv.txt') - self.virtualenv_manager.install_pip_requirements( - pipenv_reqs, require_hashes=False, vendored=True) + for package in ['certifi', 'pipenv', 'six', 'virtualenv', 'virtualenv-clone']: + path = os.path.normpath(os.path.join(self.topsrcdir, 'third_party/python', package)) + self.virtualenv_manager.install_pip_package(path, vendored=True) return pipenv - def activate_pipenv(self, path, args=None): - if not os.path.exists(path): - raise Exception('Pipfile not found: %s.' % path) + def activate_pipenv(self, pipfile=None, args=None, populate=False): + if pipfile is not None and not os.path.exists(pipfile): + raise Exception('Pipfile not found: %s.' % pipfile) self.ensure_pipenv() - self.virtualenv_manager.activate_pipenv(path, args) + self.virtualenv_manager.activate_pipenv(pipfile, args, populate) class MachCommandBase(MozbuildObject): diff --git a/python/mozbuild/mozbuild/pipenv.txt b/python/mozbuild/mozbuild/pipenv.txt deleted file mode 100644 index 45100ff890a4..000000000000 --- a/python/mozbuild/mozbuild/pipenv.txt +++ /dev/null @@ -1,5 +0,0 @@ -third_party/python/certifi -third_party/python/pipenv -third_party/python/six -third_party/python/virtualenv -third_party/python/virtualenv-clone diff --git a/python/mozbuild/mozbuild/virtualenv.py b/python/mozbuild/mozbuild/virtualenv.py index ed593bce450f..0cda77df2d9d 100644 --- a/python/mozbuild/mozbuild/virtualenv.py +++ b/python/mozbuild/mozbuild/virtualenv.py @@ -20,9 +20,11 @@ IS_NATIVE_WIN = (sys.platform == 'win32' and os.sep == '\\') IS_MSYS2 = (sys.platform == 'win32' and os.sep == '/') IS_CYGWIN = (sys.platform == 'cygwin') -# Minimum version of Python required to build. -MINIMUM_PYTHON_VERSION = LooseVersion('2.7.3') -MINIMUM_PYTHON_MAJOR = 2 +# Minimum versions of Python required to build. +MINIMUM_PYTHON_VERSIONS = { + 2: LooseVersion('2.7.3'), + 3: LooseVersion('3.5.0') +} UPGRADE_WINDOWS = ''' @@ -38,6 +40,8 @@ another Python version. Ensure a modern Python can be found in the paths defined by the $PATH environment variable and try again. '''.lstrip() +here = os.path.abspath(os.path.dirname(__file__)) + class VirtualenvManager(object): """Contains logic for managing virtualenvs for building the tree.""" @@ -207,7 +211,7 @@ class VirtualenvManager(object): return self.virtualenv_root def packages(self): - with file(self.manifest_path, 'rU') as fh: + with open(self.manifest_path, 'rU') as fh: packages = [line.rstrip().split(':') for line in fh] return packages @@ -250,7 +254,6 @@ class VirtualenvManager(object): environment is not configured properly, packages could be installed into the wrong place. This is how virtualenv's work. """ - packages = self.packages() python_lib = distutils.sysconfig.get_python_lib() @@ -465,13 +468,16 @@ class VirtualenvManager(object): if isinstance(os.environ['PATH'], unicode): os.environ['PATH'] = os.environ['PATH'].encode('utf-8') - def install_pip_package(self, package): + def install_pip_package(self, package, vendored=False): """Install a package via pip. The supplied package is specified using a pip requirement specifier. e.g. 'foo' or 'foo==1.0'. If the package is already installed, this is a no-op. + + If vendored is True, no package index will be used and no dependencies + will be installed. """ from pip.req import InstallRequirement @@ -486,6 +492,12 @@ class VirtualenvManager(object): package, ] + if vendored: + args.extend([ + '--no-deps', + '--no-index', + ]) + return self._run_pip(args) def install_pip_requirements(self, path, require_hashes=True, quiet=False, vendored=False): @@ -530,22 +542,34 @@ class VirtualenvManager(object): # force the virtualenv's interpreter to be used and all is well. # It /might/ be possible to cheat and set sys.executable to # self.python_path. However, this seems more risk than it's worth. - subprocess.check_call([os.path.join(self.bin_path, 'pip')] + args, - stderr=subprocess.STDOUT) + pip = os.path.join(self.bin_path, 'pip') + subprocess.check_call([pip] + args, stderr=subprocess.STDOUT, cwd=self.topsrcdir) - def activate_pipenv(self, pipfile, args=None): - """Install a Pipfile located at path and activate environment""" + def activate_pipenv(self, pipfile=None, args=None, populate=False): + """Activate a virtual environment managed by pipenv + + If ``pipfile`` is not ``None`` then the Pipfile located at the path + provided will be used to create the virtual environment. If + ``populate`` is ``True`` then the virtual environment will be + populated from the manifest file. The optional ``args`` list will be + passed to the pipenv commands. + """ pipenv = os.path.join(self.bin_path, 'pipenv') env = os.environ.copy() env.update({ - 'PIPENV_IGNORE_VIRTUALENVS': '1', - 'PIPENV_PIPFILE': pipfile, - 'WORKON_HOME': os.path.join(self.topobjdir, '_virtualenvs'), + b'PIPENV_IGNORE_VIRTUALENVS': b'1', + b'WORKON_HOME': str(os.path.normpath(os.path.join(self.topobjdir, '_virtualenvs'))), }) args = args or [] + + if pipfile is not None: + # Install from Pipfile + env[b'PIPENV_PIPFILE'] = str(pipfile) + args.append('install') + subprocess.check_call( - [pipenv, 'install'] + args, + [pipenv] + args, stderr=subprocess.STDOUT, env=env) @@ -554,6 +578,13 @@ class VirtualenvManager(object): stderr=subprocess.STDOUT, env=env).rstrip() + if populate: + # Populate from the manifest + subprocess.check_call([ + pipenv, 'run', 'python', os.path.join(here, 'virtualenv.py'), 'populate', + self.topsrcdir, self.topobjdir, self.virtualenv_root, self.manifest_path], + stderr=subprocess.STDOUT, env=env) + self.activate() @@ -563,9 +594,10 @@ def verify_python_version(log_handle): our = LooseVersion('%d.%d.%d' % (major, minor, micro)) - if major != MINIMUM_PYTHON_MAJOR or our < MINIMUM_PYTHON_VERSION: - log_handle.write('Python %s or greater (but not Python 3) is ' - 'required to build. ' % MINIMUM_PYTHON_VERSION) + if major not in MINIMUM_PYTHON_VERSIONS or our < MINIMUM_PYTHON_VERSIONS[major]: + log_handle.write('One of the following Python versions are required to build:\n') + for minver in MINIMUM_PYTHON_VERSIONS.values(): + log_handle.write('* Python %s or greater\n' % minver) log_handle.write('You are running Python %s.\n' % our) if os.name in ('nt', 'ce'):