mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-10-10 03:45:46 +00:00
Bug 1590745 - Make the $PYTHON3 build var use a virtualenv r=mshal
Make the $PYTHON3 build var point to a full virtualenv bootstrapped with the same libraries as the $PYTHON Python 2 build var. This allows us to upgrade build tasks from $PYTHON to $PYTHON3. This patch adds some debug logging and documentation to the Python 2 virtualenv so that it is easier to diagnose issues that may arise from running two different Python interpreters in re-entrant multiprocess routines. Differential Revision: https://phabricator.services.mozilla.com/D50819 --HG-- extra : moz-landing-system : lando
This commit is contained in:
Māris Fogels
parent
8200f05c80
commit
75d85af992
@ -196,26 +196,40 @@ def mozconfig(mozconfig, old_configure, build_env,
|
||||
set_config('MOZCONFIG', depends(mozconfig)(lambda m: m['path']))
|
||||
|
||||
|
||||
option(env='PYTHON', nargs=1, help='Python interpreter')
|
||||
# Python 2
|
||||
# ========
|
||||
|
||||
# Setup python virtualenv
|
||||
# ==============================================================
|
||||
option(env='PYTHON', nargs=1, help='Python 2.7 interpreter')
|
||||
|
||||
|
||||
@depends('PYTHON', check_build_environment, mozconfig, '--help')
|
||||
@imports('os')
|
||||
@imports('sys')
|
||||
@imports('subprocess')
|
||||
@imports('distutils.sysconfig')
|
||||
@imports(_from='mozbuild.configure.util', _import='LineIO')
|
||||
@imports(_from='mozbuild.virtualenv', _import='VirtualenvManager')
|
||||
@imports(_from='mozbuild.virtualenv', _import='verify_python_version')
|
||||
@imports('distutils.sysconfig')
|
||||
def virtualenv_python(env_python, build_env, mozconfig, help):
|
||||
@imports(_from='mozbuild.virtualenv', _import='PY2')
|
||||
def virtualenv_python2(env_python, build_env, mozconfig, help):
|
||||
if help:
|
||||
return
|
||||
|
||||
# NOTE: We cannot assume the Python we are calling this code with is the
|
||||
# Python we want to set up a virtualenv for.
|
||||
#
|
||||
# We also cannot assume that the Python the caller is configuring meets our
|
||||
# build requirements.
|
||||
#
|
||||
# Because of this the code is written to re-execute itself with the correct
|
||||
# interpreter if required.
|
||||
|
||||
log.debug("python2: running with pid %r" % os.getpid())
|
||||
log.debug("python2: sys.executable: %r" % sys.executable)
|
||||
|
||||
python = env_python[0] if env_python else None
|
||||
|
||||
# Did our python come from mozconfig? Overrides environment setting.
|
||||
# Ideally we'd rely on the mozconfig injection from mozconfig_options,
|
||||
# but we'd rather avoid the verbosity when we need to reexecute with
|
||||
# a different python.
|
||||
@ -229,6 +243,8 @@ def virtualenv_python(env_python, build_env, mozconfig, help):
|
||||
elif 'PYTHON' in mozconfig['vars']['modified']:
|
||||
python = mozconfig['vars']['modified']['PYTHON'][1]
|
||||
|
||||
log.debug("python2: executable from configuration: %r" % python)
|
||||
|
||||
with LineIO(lambda l: log.error(l)) as out:
|
||||
verify_python_version(out)
|
||||
topsrcdir, topobjdir = build_env.topsrcdir, build_env.topobjdir
|
||||
@ -256,29 +272,42 @@ def virtualenv_python(env_python, build_env, mozconfig, help):
|
||||
else:
|
||||
python = sys.executable
|
||||
|
||||
log.debug("python2: found executable: %r" % python)
|
||||
|
||||
if not manager.up_to_date(python):
|
||||
log.info('Creating Python environment')
|
||||
log.info('Creating Python 2 environment')
|
||||
manager.build(python)
|
||||
else:
|
||||
log.debug("python2: venv is up to date")
|
||||
|
||||
python = normsep(manager.python_path)
|
||||
|
||||
if not normsep(sys.executable).startswith(normsep(virtualenvs_root)):
|
||||
log.info('Reexecuting in the virtualenv')
|
||||
if env_python:
|
||||
del os.environ['PYTHON']
|
||||
# One would prefer to use os.execl, but that's completely borked on
|
||||
# Windows.
|
||||
sys.exit(subprocess.call([python] + sys.argv))
|
||||
# The currently running interpreter could be Python 2 or Python 3. We make the
|
||||
# part of the code that re-executes everything with the virtualenv's Python
|
||||
# conditional on running the same major version as the current interpreter. If we
|
||||
# don't do this then the configure code for the Py 2 and Py 3 virtualenvs could
|
||||
# activate each other from inside the other's virtualenv. We can't guarantee
|
||||
# how the virtualenvs would interact if that happens.
|
||||
if PY2:
|
||||
if not normsep(sys.executable).startswith(normsep(virtualenvs_root)):
|
||||
log.debug("python2: executing as %s, should be running as %s" % (
|
||||
sys.executable, manager.python_path))
|
||||
log.info('Reexecuting in the virtualenv')
|
||||
if env_python:
|
||||
del os.environ['PYTHON']
|
||||
# One would prefer to use os.execl, but that's completely borked on
|
||||
# Windows.
|
||||
sys.exit(subprocess.call([python] + sys.argv))
|
||||
|
||||
# We are now in the virtualenv
|
||||
if not distutils.sysconfig.get_python_lib():
|
||||
die('Could not determine python site packages directory')
|
||||
# We are now in the virtualenv
|
||||
if not distutils.sysconfig.get_python_lib():
|
||||
die('Could not determine python site packages directory')
|
||||
|
||||
return python
|
||||
|
||||
|
||||
set_config('PYTHON', virtualenv_python)
|
||||
add_old_configure_assignment('PYTHON', virtualenv_python)
|
||||
set_config('PYTHON', virtualenv_python2)
|
||||
add_old_configure_assignment('PYTHON', virtualenv_python2)
|
||||
|
||||
# Inject mozconfig options
|
||||
# ==============================================================
|
||||
@ -386,16 +415,71 @@ shell = help_shell | shell
|
||||
option(env='PYTHON3', nargs=1, help='Python 3 interpreter (3.5 or later)')
|
||||
|
||||
|
||||
@depends('PYTHON3', 'MOZILLABUILD')
|
||||
@depends(
|
||||
'PYTHON3', check_build_environment, 'MOZILLABUILD', mozconfig, '--help')
|
||||
@checking('for Python 3',
|
||||
callback=lambda x: '%s (%s)' % (x.path, x.str_version) if x else 'no')
|
||||
@imports(_from='__builtin__', _import='Exception')
|
||||
@imports('os')
|
||||
@imports('sys')
|
||||
@imports('subprocess')
|
||||
@imports('distutils.sysconfig')
|
||||
@imports(_from='mozbuild.configure.util', _import='LineIO')
|
||||
@imports(_from='mozbuild.virtualenv', _import='VirtualenvManager')
|
||||
@imports(_from='mozbuild.virtualenv', _import='verify_python_version')
|
||||
@imports(_from='mozbuild.virtualenv', _import='PY3')
|
||||
@imports(_from='mozbuild.pythonutil', _import='find_python3_executable')
|
||||
@imports(_from='mozbuild.pythonutil', _import='python_executable_version')
|
||||
def python3(env_python, mozillabuild):
|
||||
def virtualenv_python3(env_python, build_env, mozillabuild, mozconfig, help):
|
||||
if help:
|
||||
return
|
||||
|
||||
# NOTE: We cannot assume the Python we are calling this code with is the
|
||||
# Python we want to set up a virtualenv for.
|
||||
#
|
||||
# We also cannot assume that the Python the caller is configuring meets our
|
||||
# build requirements.
|
||||
#
|
||||
# Because of this the code is written to re-execute itself with the correct
|
||||
# interpreter if required.
|
||||
|
||||
log.debug("python3: running with pid %r" % os.getpid())
|
||||
log.debug("python3: sys.executable: %r" % sys.executable)
|
||||
|
||||
# Verify that the Python version we executed this code with is the minimum
|
||||
# required version to handle all project code.
|
||||
with LineIO(lambda l: log.error(l)) as out:
|
||||
verify_python_version(out)
|
||||
|
||||
python = env_python[0] if env_python else None
|
||||
|
||||
# If Python given by environment variable, it must work.
|
||||
# Ideally we'd rely on the mozconfig injection from mozconfig_options,
|
||||
# but we'd rather avoid the verbosity when we need to reexecute with
|
||||
# a different python.
|
||||
if mozconfig['path']:
|
||||
if 'PYTHON3' in mozconfig['env']['added']:
|
||||
python = mozconfig['env']['added']['PYTHON3']
|
||||
elif 'PYTHON3' in mozconfig['env']['modified']:
|
||||
python = mozconfig['env']['modified']['PYTHON3'][1]
|
||||
elif 'PYTHON3' in mozconfig['vars']['added']:
|
||||
python = mozconfig['vars']['added']['PYTHON3']
|
||||
elif 'PYTHON3' in mozconfig['vars']['modified']:
|
||||
python = mozconfig['vars']['modified']['PYTHON3'][1]
|
||||
|
||||
log.debug("python3: executable from configuration: %r" % python)
|
||||
|
||||
# If this is a mozilla-central build, we'll find the virtualenv in the top
|
||||
# source directory. If this is a SpiderMonkey build, we assume we're at
|
||||
# js/src and try to find the virtualenv from the mozilla-central root.
|
||||
# See mozilla-central changeset d2cce982a7c809815d86d5daecefe2e7a563ecca
|
||||
# Bug 784841
|
||||
topsrcdir, topobjdir = build_env.topsrcdir, build_env.topobjdir
|
||||
if topobjdir.endswith('/js/src'):
|
||||
topobjdir = topobjdir[:-7]
|
||||
|
||||
# If we know the Python executable the caller is asking for then verify its
|
||||
# version. If the caller did not ask for a specific executable then find
|
||||
# a reasonable default.
|
||||
if python:
|
||||
try:
|
||||
version = python_executable_version(python).version
|
||||
@ -430,15 +514,60 @@ def python3(env_python, mozillabuild):
|
||||
'%s is Python %d.%d' % (python, version[0],
|
||||
version[1]))
|
||||
|
||||
log.debug("python3: found executable: %r" % python)
|
||||
|
||||
virtualenvs_root = os.path.join(topobjdir, '_virtualenvs')
|
||||
with LineIO(lambda l: log.info(l), 'replace') as out:
|
||||
manager = VirtualenvManager(
|
||||
topsrcdir, topobjdir,
|
||||
os.path.join(virtualenvs_root, 'init_py3'), out,
|
||||
os.path.join(topsrcdir, 'build', 'virtualenv_packages.txt'))
|
||||
|
||||
log.debug("python3: using venv: %r" % manager.virtualenv_root)
|
||||
|
||||
if not manager.up_to_date(python):
|
||||
log.info('Creating Python 3 environment')
|
||||
manager.build(python)
|
||||
else:
|
||||
log.debug("python3: venv is up to date")
|
||||
|
||||
python = normsep(manager.python_path)
|
||||
|
||||
# The currently running interpreter could be Python 2 or Python 3. We make the
|
||||
# part of the code that re-executes everything with the virtualenv's Python
|
||||
# conditional on running the same major version as the current interpreter. If we
|
||||
# don't do this then the configure code for the Py 2 and Py 3 virtualenvs could
|
||||
# activate each other from inside the other's virtualenv. We can't guarantee
|
||||
# how the virtualenvs would interact if that happens.
|
||||
if PY3:
|
||||
if not normsep(sys.executable).startswith(normsep(virtualenvs_root)):
|
||||
log.debug("python3: executing as %s, should be running as %s" % (
|
||||
sys.executable, manager.python_path))
|
||||
log.info('Re-executing in the virtualenv')
|
||||
if env_python:
|
||||
del os.environ['PYTHON3']
|
||||
# One would prefer to use os.execl, but that's completely borked on
|
||||
# Windows.
|
||||
sys.exit(subprocess.call([python] + sys.argv))
|
||||
|
||||
# We are now in the virtualenv
|
||||
if not distutils.sysconfig.get_python_lib():
|
||||
die('Could not determine python site packages directory')
|
||||
|
||||
str_version = '.'.join(str(v) for v in version)
|
||||
|
||||
return namespace(
|
||||
path=python,
|
||||
version=version,
|
||||
str_version='.'.join(str(v) for v in version),
|
||||
str_version=str_version,
|
||||
)
|
||||
|
||||
|
||||
set_config('PYTHON3', depends_if(python3)(lambda p: p.path))
|
||||
set_config('PYTHON3_VERSION', depends_if(python3)(lambda p: p.str_version))
|
||||
set_config('PYTHON3', depends(virtualenv_python3)(lambda p: p.path))
|
||||
set_config(
|
||||
'PYTHON3_VERSION',
|
||||
depends(virtualenv_python3)(lambda p: p.str_version))
|
||||
|
||||
|
||||
# Source checkout and version control integration.
|
||||
# ================================================
|
||||
|
||||
@ -25,6 +25,7 @@ from manifestparser import filters as mpf
|
||||
from mozbuild.base import (
|
||||
MachCommandBase,
|
||||
)
|
||||
from mozbuild.virtualenv import VirtualenvManager
|
||||
|
||||
from mach.decorators import (
|
||||
CommandArgument,
|
||||
@ -138,8 +139,7 @@ class MachCommands(MachCommandBase):
|
||||
exitfirst=False,
|
||||
extra=None,
|
||||
**kwargs):
|
||||
python = python or self.virtualenv_manager.python_path
|
||||
self.activate_pipenv(pipfile=None, populate=True, python=python)
|
||||
self._activate_test_virtualenvs(python)
|
||||
|
||||
if test_objects is None:
|
||||
from moztest.resolve import TestResolver
|
||||
@ -231,6 +231,38 @@ class MachCommands(MachCommandBase):
|
||||
'Return code from mach python-test: {return_code}')
|
||||
return return_code
|
||||
|
||||
def _activate_test_virtualenvs(self, python):
|
||||
"""Make sure the test suite virtualenvs are set up and activated.
|
||||
|
||||
Args:
|
||||
python: Optional python version string we want to run the suite with.
|
||||
See the `--python` argument to the `mach python-test` command.
|
||||
"""
|
||||
from mozbuild.pythonutil import find_python3_executable
|
||||
|
||||
default_manager = self.virtualenv_manager
|
||||
|
||||
# Grab the default virtualenv properties before we activate other virtualenvs.
|
||||
python = python or default_manager.python_path
|
||||
py3_root = default_manager.virtualenv_root + '_py3'
|
||||
|
||||
self.activate_pipenv(pipfile=None, populate=True, python=python)
|
||||
|
||||
# The current process might be running under Python 2 and the Python 3
|
||||
# virtualenv will not be set up by mach bootstrap. To avoid problems in tests
|
||||
# that implicitly depend on the Python 3 virtualenv we ensure the Python 3
|
||||
# virtualenv is up to date before the tests start.
|
||||
python3, version = find_python3_executable(min_version='3.5.0')
|
||||
|
||||
py3_manager = VirtualenvManager(
|
||||
default_manager.topsrcdir,
|
||||
default_manager.topobjdir,
|
||||
py3_root,
|
||||
default_manager.log_handle,
|
||||
default_manager.manifest_path,
|
||||
)
|
||||
py3_manager.ensure(python3)
|
||||
|
||||
def _run_python_test(self, test):
|
||||
from mozprocess import ProcessHandler
|
||||
|
||||
|
||||
@ -13,8 +13,7 @@ import traceback
|
||||
from textwrap import dedent
|
||||
|
||||
from mozpack import path as mozpath
|
||||
from mozbuild.util import system_encoding
|
||||
|
||||
from mozbuild.util import system_encoding, ensure_subprocess_env
|
||||
|
||||
MOZ_MYCONFIG_ERROR = '''
|
||||
The MOZ_MYCONFIG environment variable to define the location of mozconfigs
|
||||
@ -238,8 +237,6 @@ class MozconfigLoader(object):
|
||||
result['make_extra'] = []
|
||||
result['make_flags'] = []
|
||||
|
||||
env = dict(os.environ)
|
||||
|
||||
# Since mozconfig_loader is a shell script, running it "normally"
|
||||
# actually leads to two shell executions on Windows. Avoid this by
|
||||
# directly calling sh mozconfig_loader.
|
||||
@ -258,7 +255,8 @@ class MozconfigLoader(object):
|
||||
# We need to capture stderr because that's where the shell sends
|
||||
# errors if execution fails.
|
||||
output = subprocess.check_output(command, stderr=subprocess.STDOUT,
|
||||
cwd=self.topsrcdir, env=env)
|
||||
cwd=self.topsrcdir,
|
||||
env=ensure_subprocess_env(os.environ))
|
||||
except subprocess.CalledProcessError as e:
|
||||
lines = e.output.splitlines()
|
||||
|
||||
|
||||
@ -118,14 +118,26 @@ class VirtualenvManager(object):
|
||||
on OS X our python path may end up being a different or modified
|
||||
executable.
|
||||
"""
|
||||
ver = subprocess.check_output([python, '-c', 'import sys; print(sys.hexversion)'],
|
||||
universal_newlines=True).rstrip()
|
||||
ver = self.python_executable_hexversion(python)
|
||||
with open(self.exe_info_path, 'w') as fh:
|
||||
fh.write("%s\n" % ver)
|
||||
fh.write("%s\n" % os.path.getsize(python))
|
||||
|
||||
def up_to_date(self, python=sys.executable):
|
||||
"""Returns whether the virtualenv is present and up to date."""
|
||||
def python_executable_hexversion(self, python):
|
||||
"""Run a Python executable and return its sys.hexversion value."""
|
||||
program = 'import sys; print(sys.hexversion)'
|
||||
out = subprocess.check_output([python, '-c', program]).rstrip()
|
||||
return int(out)
|
||||
|
||||
def up_to_date(self, python):
|
||||
"""Returns whether the virtualenv is present and up to date.
|
||||
|
||||
Args:
|
||||
python: Full path string to the Python executable that this virtualenv
|
||||
should be running. If the Python executable passed in to this
|
||||
argument is not the same version as the Python the virtualenv was
|
||||
built with then this method will return False.
|
||||
"""
|
||||
|
||||
deps = [self.manifest_path, __file__]
|
||||
|
||||
@ -134,7 +146,8 @@ class VirtualenvManager(object):
|
||||
not os.path.exists(self.activate_path):
|
||||
return False
|
||||
|
||||
# check modification times
|
||||
# Modifications to our package dependency list or to this file mean the
|
||||
# virtualenv should be rebuilt.
|
||||
activate_mtime = os.path.getmtime(self.activate_path)
|
||||
dep_mtime = max(os.path.getmtime(p) for p in deps)
|
||||
if dep_mtime > activate_mtime:
|
||||
@ -144,9 +157,11 @@ class VirtualenvManager(object):
|
||||
# python, or we have the Python version that was used to create the
|
||||
# virtualenv. If this fails, it is likely system Python has been
|
||||
# upgraded, and our virtualenv would not be usable.
|
||||
orig_version, orig_size = self.get_exe_info()
|
||||
python_size = os.path.getsize(python)
|
||||
hexversion = self.python_executable_hexversion(python)
|
||||
if ((python, python_size) != (self.python_path, os.path.getsize(self.python_path)) and
|
||||
(sys.hexversion, python_size) != self.get_exe_info()):
|
||||
(hexversion, python_size) != (orig_version, orig_size)):
|
||||
return False
|
||||
|
||||
# recursively check sub packages.txt files
|
||||
@ -193,14 +208,13 @@ class VirtualenvManager(object):
|
||||
|
||||
return proc.wait()
|
||||
|
||||
def create(self, python=sys.executable):
|
||||
def create(self, python):
|
||||
"""Create a new, empty virtualenv.
|
||||
|
||||
Receives the path to virtualenv's virtualenv.py script (which will be
|
||||
called out to), the path to create the virtualenv in, and a handle to
|
||||
write output to.
|
||||
"""
|
||||
env = dict(os.environ)
|
||||
|
||||
args = [python, self.virtualenv_script_path,
|
||||
# Without this, virtualenv.py may attempt to contact the outside
|
||||
@ -210,11 +224,13 @@ class VirtualenvManager(object):
|
||||
'--no-download',
|
||||
self.virtualenv_root]
|
||||
|
||||
result = self._log_process_output(args, env=env)
|
||||
result = self._log_process_output(args,
|
||||
env=ensure_subprocess_env(os.environ))
|
||||
|
||||
if result:
|
||||
raise Exception(
|
||||
'Failed to create virtualenv: %s' % self.virtualenv_root)
|
||||
'Failed to create virtualenv: %s (virtualenv.py retcode: %s)' % (
|
||||
self.virtualenv_root, result))
|
||||
|
||||
self.write_exe_info(python)
|
||||
|
||||
@ -468,7 +484,7 @@ class VirtualenvManager(object):
|
||||
|
||||
raise Exception('Error installing package: %s' % directory)
|
||||
|
||||
def build(self, python=sys.executable):
|
||||
def build(self, python):
|
||||
"""Build a virtualenv per tree conventions.
|
||||
|
||||
This returns the path of the created virtualenv.
|
||||
@ -479,7 +495,16 @@ class VirtualenvManager(object):
|
||||
# We need to populate the virtualenv using the Python executable in
|
||||
# the virtualenv for paths to be proper.
|
||||
|
||||
args = [self.python_path, __file__, 'populate', self.topsrcdir,
|
||||
# If this module was run from Python 2 then the __file__ attribute may
|
||||
# point to a Python 2 .pyc file. If we are generating a Python 3
|
||||
# virtualenv from Python 2 make sure we call Python 3 with the path to
|
||||
# the module and not the Python 2 .pyc file.
|
||||
if os.path.splitext(__file__)[1] in ('.pyc', '.pyo'):
|
||||
thismodule = __file__[:-1]
|
||||
else:
|
||||
thismodule = __file__
|
||||
|
||||
args = [self.python_path, thismodule, 'populate', self.topsrcdir,
|
||||
self.topobjdir, self.virtualenv_root, self.manifest_path]
|
||||
|
||||
result = self._log_process_output(args, cwd=self.topsrcdir)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user