Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-27 23:02:20 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1466211 - Switch all |mach python-test| tests to run using pipenv; r=ahal

Browse Source 
MozReview-Commit-ID: AzmdDgAgZgI

--HG--
extra : rebase_source : cd4603b707bd4ca3f7e0783cf5b9adb8c47a9e01
extra : source : e550be515be30148c0a89c33953dff4148a6a171
This commit is contained in:
Dave Hunt 2018-06-08 13:24:27 +01:00
parent 78119bf63a
commit 3a2583759b
8 changed files with 60 additions and 184 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Pipfile
View File

@ -16,7 +16,3 @@ requests = "==2.9.1"
six = "==1.10.0"
virtualenv = "==15.2.0"
voluptuous = "==0.10.5"
[requires]
python_version = "2.7"

6
Pipfile.lock generated
View File

@ -1,12 +1,10 @@
{
"_meta": {
"hash": {
"sha256": "af4e239c88ce3d74e2e3dd7d352c3e8a203ce476c7369b2a4dc0eea7114996ba"
"sha256": "eb8b0a9771d4420f83fbbabf9952dc783aeefe9be455559de2f3ebff27caa93f"
},
"pipfile-spec": 6,
"requires": {
"python_version": "2.7"
},
"requires": {},
"sources": [
{
"name": "pypi",

36
python/Pipfile
View File

@ -1,36 +0,0 @@
[[source]]
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"
[packages]
"d5b4a14" = {path = "./mach"}
"8ddb376" = {path = "./mozbuild"}
"b3ddbcf" = {path = "./mozterm"}
"38a4a9a" = {path = "./mozversioncontrol"}
"26d92fb" = {path = "./../config/mozunit"}
"cea2946" = {path = "./../testing/mozbase/manifestparser"}
"ffcf6e6" = {path = "./../testing/mozbase/mozcrash"}
"195ae2e" = {path = "./../testing/mozbase/mozdebug"}
"8dab59a" = {path = "./../testing/mozbase/mozdevice"}
"58d0848" = {path = "./../testing/mozbase/mozfile"}
"fd0b608" = {path = "./../testing/mozbase/mozhttpd"}
"7329809" = {path = "./../testing/mozbase/mozinfo"}
"501835d" = {path = "./../testing/mozbase/mozinstall"}
"807c1c5" = {path = "./../testing/mozbase/mozlog"}
"e09e103" = {path = "./../testing/mozbase/moznetwork"}
"132adec" = {path = "./../testing/mozbase/mozprocess"}
"d88f467" = {path = "./../testing/mozbase/mozprofile"}
"1de94f2" = {path = "./../testing/mozbase/mozrunner"}
"6477f20" = {path = "./../testing/mozbase/moztest"}
"f1d74ca" = {path = "./../testing/mozbase/mozversion"}
"47200d8" = {path = "./../third_party/python/futures", markers="python_version < '3'"}
"110bcc4" = {path = "./../third_party/python/jsmin"}
"c49d32a" = {path = "./../third_party/python/mock-1.0.0", markers="python_version < '3.3'"}
"c2c21d9" = {path = "./../third_party/python/py"}
"f4b00e9" = {path = "./../third_party/python/pytest"}
"053111f" = {path = "./../third_party/python/requests"}
"d250320" = {path = "./../third_party/python/six"}
"f1de77a" = {path = "./../third_party/python/which", markers="python_version < '3.3'"}
[dev-packages]

106
python/Pipfile.lock generated
View File

@ -1,106 +0,0 @@
{
"_meta": {
"hash": {
"sha256": "dfc219f64edc7715acdb35e03dcee665ec26908c18a58d3a3a88dda3ab393b17"
},
"pipfile-spec": 6,
"requires": {},
"sources": [
{
"name": "pypi",
"url": "https://pypi.org/simple",
"verify_ssl": true
}
]
},
"default": {
"053111f": {
"path": "./../third_party/python/requests"
},
"110bcc4": {
"path": "./../third_party/python/jsmin"
},
"132adec": {
"path": "./../testing/mozbase/mozprocess"
},
"195ae2e": {
"path": "./../testing/mozbase/mozdebug"
},
"1de94f2": {
"path": "./../testing/mozbase/mozrunner"
},
"26d92fb": {
"path": "./../config/mozunit"
},
"38a4a9a": {
"path": "./mozversioncontrol"
},
"47200d8": {
"markers": "python_version < '3'",
"path": "./../third_party/python/futures"
},
"501835d": {
"path": "./../testing/mozbase/mozinstall"
},
"58d0848": {
"path": "./../testing/mozbase/mozfile"
},
"6477f20": {
"path": "./../testing/mozbase/moztest"
},
"7329809": {
"path": "./../testing/mozbase/mozinfo"
},
"807c1c5": {
"path": "./../testing/mozbase/mozlog"
},
"8dab59a": {
"path": "./../testing/mozbase/mozdevice"
},
"8ddb376": {
"path": "./mozbuild"
},
"b3ddbcf": {
"path": "./mozterm"
},
"c2c21d9": {
"path": "./../third_party/python/py"
},
"c49d32a": {
"markers": "python_version < '3.3'",
"path": "./../third_party/python/mock-1.0.0"
},
"cea2946": {
"path": "./../testing/mozbase/manifestparser"
},
"d250320": {
"path": "./../third_party/python/six"
},
"d5b4a14": {
"path": "./mach"
},
"d88f467": {
"path": "./../testing/mozbase/mozprofile"
},
"e09e103": {
"path": "./../testing/mozbase/moznetwork"
},
"f1d74ca": {
"path": "./../testing/mozbase/mozversion"
},
"f1de77a": {
"markers": "python_version < '3.3'",
"path": "./../third_party/python/which"
},
"f4b00e9": {
"path": "./../third_party/python/pytest"
},
"fd0b608": {
"path": "./../testing/mozbase/mozhttpd"
},
"ffcf6e6": {
"path": "./../testing/mozbase/mozcrash"
}
},
"develop": {}
}

7
python/mach_commands.py
View File

@ -99,11 +99,8 @@ class MachCommands(MachCommandBase):
jobs=1,
three=False,
**kwargs):
if three:
# use pipenv to run tests against Python 3
self.activate_pipenv(os.path.join(here, 'Pipfile'), ['--three'])
else:
self._activate_virtualenv()
pipenv_args = ['--three' if three else '--two']
self.activate_pipenv(pipfile=None, args=pipenv_args, populate=True)
if test_objects is None:
from moztest.resolve import TestResolver

14
python/mozbuild/mozbuild/base.py
View File

@ -752,16 +752,16 @@ class MozbuildObject(ProcessExecutionMixin):
self._activate_virtualenv()
pipenv = os.path.join(self.virtualenv_manager.bin_path, 'pipenv')
if not os.path.exists(pipenv):
pipenv_reqs = os.path.join(self.topsrcdir, 'python/mozbuild/mozbuild/pipenv.txt')
self.virtualenv_manager.install_pip_requirements(
pipenv_reqs, require_hashes=False, vendored=True)
for package in ['certifi', 'pipenv', 'six', 'virtualenv', 'virtualenv-clone']:
path = os.path.normpath(os.path.join(self.topsrcdir, 'third_party/python', package))
self.virtualenv_manager.install_pip_package(path, vendored=True)
return pipenv
def activate_pipenv(self, path, args=None):
if not os.path.exists(path):
raise Exception('Pipfile not found: %s.' % path)
def activate_pipenv(self, pipfile=None, args=None, populate=False):
if pipfile is not None and not os.path.exists(pipfile):
raise Exception('Pipfile not found: %s.' % pipfile)
self.ensure_pipenv()
self.virtualenv_manager.activate_pipenv(path, args)
self.virtualenv_manager.activate_pipenv(pipfile, args, populate)
class MachCommandBase(MozbuildObject):

5
python/mozbuild/mozbuild/pipenv.txt
View File

@ -1,5 +0,0 @@
third_party/python/certifi
third_party/python/pipenv
third_party/python/six
third_party/python/virtualenv
third_party/python/virtualenv-clone

66
python/mozbuild/mozbuild/virtualenv.py
View File

@ -20,9 +20,11 @@ IS_NATIVE_WIN = (sys.platform == 'win32' and os.sep == '\\')
IS_MSYS2 = (sys.platform == 'win32' and os.sep == '/')
IS_CYGWIN = (sys.platform == 'cygwin')
# Minimum version of Python required to build.
MINIMUM_PYTHON_VERSION = LooseVersion('2.7.3')
MINIMUM_PYTHON_MAJOR = 2
# Minimum versions of Python required to build.
MINIMUM_PYTHON_VERSIONS = {
2: LooseVersion('2.7.3'),
3: LooseVersion('3.5.0')
}
UPGRADE_WINDOWS = '''
@ -38,6 +40,8 @@ another Python version. Ensure a modern Python can be found in the paths
defined by the $PATH environment variable and try again.
'''.lstrip()
here = os.path.abspath(os.path.dirname(__file__))
class VirtualenvManager(object):
"""Contains logic for managing virtualenvs for building the tree."""
@ -207,7 +211,7 @@ class VirtualenvManager(object):
return self.virtualenv_root
def packages(self):
with file(self.manifest_path, 'rU') as fh:
with open(self.manifest_path, 'rU') as fh:
packages = [line.rstrip().split(':')
for line in fh]
return packages
@ -250,7 +254,6 @@ class VirtualenvManager(object):
environment is not configured properly, packages could be installed
into the wrong place. This is how virtualenv's work.
"""
packages = self.packages()
python_lib = distutils.sysconfig.get_python_lib()
@ -465,13 +468,16 @@ class VirtualenvManager(object):
if isinstance(os.environ['PATH'], unicode):
os.environ['PATH'] = os.environ['PATH'].encode('utf-8')
def install_pip_package(self, package):
def install_pip_package(self, package, vendored=False):
"""Install a package via pip.
The supplied package is specified using a pip requirement specifier.
e.g. 'foo' or 'foo==1.0'.
If the package is already installed, this is a no-op.
If vendored is True, no package index will be used and no dependencies
will be installed.
"""
from pip.req import InstallRequirement
@ -486,6 +492,12 @@ class VirtualenvManager(object):
package,
]
if vendored:
args.extend([
'--no-deps',
'--no-index',
])
return self._run_pip(args)
def install_pip_requirements(self, path, require_hashes=True, quiet=False, vendored=False):
@ -530,22 +542,34 @@ class VirtualenvManager(object):
# force the virtualenv's interpreter to be used and all is well.
# It /might/ be possible to cheat and set sys.executable to
# self.python_path. However, this seems more risk than it's worth.
subprocess.check_call([os.path.join(self.bin_path, 'pip')] + args,
stderr=subprocess.STDOUT)
pip = os.path.join(self.bin_path, 'pip')
subprocess.check_call([pip] + args, stderr=subprocess.STDOUT, cwd=self.topsrcdir)
def activate_pipenv(self, pipfile, args=None):
"""Install a Pipfile located at path and activate environment"""
def activate_pipenv(self, pipfile=None, args=None, populate=False):
"""Activate a virtual environment managed by pipenv
If ``pipfile`` is not ``None`` then the Pipfile located at the path
provided will be used to create the virtual environment. If
``populate`` is ``True`` then the virtual environment will be
populated from the manifest file. The optional ``args`` list will be
passed to the pipenv commands.
"""
pipenv = os.path.join(self.bin_path, 'pipenv')
env = os.environ.copy()
env.update({
'PIPENV_IGNORE_VIRTUALENVS': '1',
'PIPENV_PIPFILE': pipfile,
'WORKON_HOME': os.path.join(self.topobjdir, '_virtualenvs'),
b'PIPENV_IGNORE_VIRTUALENVS': b'1',
b'WORKON_HOME': str(os.path.normpath(os.path.join(self.topobjdir, '_virtualenvs'))),
})
args = args or []
if pipfile is not None:
# Install from Pipfile
env[b'PIPENV_PIPFILE'] = str(pipfile)
args.append('install')
subprocess.check_call(
[pipenv, 'install'] + args,
[pipenv] + args,
stderr=subprocess.STDOUT,
env=env)
@ -554,6 +578,13 @@ class VirtualenvManager(object):
stderr=subprocess.STDOUT,
env=env).rstrip()
if populate:
# Populate from the manifest
subprocess.check_call([
pipenv, 'run', 'python', os.path.join(here, 'virtualenv.py'), 'populate',
self.topsrcdir, self.topobjdir, self.virtualenv_root, self.manifest_path],
stderr=subprocess.STDOUT, env=env)
self.activate()
@ -563,9 +594,10 @@ def verify_python_version(log_handle):
our = LooseVersion('%d.%d.%d' % (major, minor, micro))
if major != MINIMUM_PYTHON_MAJOR or our < MINIMUM_PYTHON_VERSION:
log_handle.write('Python %s or greater (but not Python 3) is '
'required to build. ' % MINIMUM_PYTHON_VERSION)
if major not in MINIMUM_PYTHON_VERSIONS or our < MINIMUM_PYTHON_VERSIONS[major]:
log_handle.write('One of the following Python versions are required to build:\n')
for minver in MINIMUM_PYTHON_VERSIONS.values():
log_handle.write('* Python %s or greater\n' % minver)
log_handle.write('You are running Python %s.\n' % our)
if os.name in ('nt', 'ce'):