mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-10-31 14:15:30 +00:00
386 lines
14 KiB
Python
Executable File
386 lines
14 KiB
Python
Executable File
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
# This file contains code for populating the virtualenv environment for
|
|
# Mozilla's build system. It is typically called as part of configure.
|
|
|
|
from __future__ import print_function, unicode_literals, with_statement
|
|
|
|
import distutils.sysconfig
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
|
|
|
|
# Minimum version of Python required to build.
|
|
MINIMUM_PYTHON_MAJOR = 2
|
|
MINIMUM_PYTHON_MINOR = 7
|
|
|
|
|
|
class VirtualenvManager(object):
|
|
"""Contains logic for managing virtualenvs for building the tree."""
|
|
|
|
def __init__(self, topsrcdir, topobjdir, virtualenv_path, log_handle,
|
|
manifest_path):
|
|
"""Create a new manager.
|
|
|
|
Each manager is associated with a source directory, a path where you
|
|
want the virtualenv to be created, and a handle to write output to.
|
|
"""
|
|
assert os.path.isabs(manifest_path), "manifest_path must be an absolute path: %s" % (manifest_path)
|
|
self.topsrcdir = topsrcdir
|
|
self.topobjdir = topobjdir
|
|
self.virtualenv_root = virtualenv_path
|
|
self.log_handle = log_handle
|
|
self.manifest_path = manifest_path
|
|
|
|
@property
|
|
def virtualenv_script_path(self):
|
|
"""Path to virtualenv's own populator script."""
|
|
return os.path.join(self.topsrcdir, 'python', 'virtualenv',
|
|
'virtualenv.py')
|
|
|
|
@property
|
|
def python_path(self):
|
|
if sys.platform in ('win32', 'cygwin'):
|
|
return os.path.join(self.virtualenv_root, 'Scripts', 'python.exe')
|
|
|
|
return os.path.join(self.virtualenv_root, 'bin', 'python')
|
|
|
|
@property
|
|
def activate_path(self):
|
|
if sys.platform in ('win32', 'cygwin'):
|
|
return os.path.join(self.virtualenv_root, 'Scripts',
|
|
'activate_this.py')
|
|
|
|
return os.path.join(self.virtualenv_root, 'bin', 'activate_this.py')
|
|
|
|
def up_to_date(self):
|
|
"""Returns whether the virtualenv is present and up to date."""
|
|
|
|
deps = [self.manifest_path, __file__]
|
|
|
|
# check if virtualenv exists
|
|
if not os.path.exists(self.virtualenv_root) or \
|
|
not os.path.exists(self.activate_path):
|
|
|
|
return False
|
|
|
|
# check modification times
|
|
activate_mtime = os.path.getmtime(self.activate_path)
|
|
dep_mtime = max(os.path.getmtime(p) for p in deps)
|
|
if dep_mtime > activate_mtime:
|
|
return False
|
|
|
|
# recursively check sub packages.txt files
|
|
submanifests = [i[1] for i in self.packages()
|
|
if i[0] == 'packages.txt']
|
|
for submanifest in submanifests:
|
|
submanifest = os.path.join(self.topsrcdir, submanifest)
|
|
submanager = VirtualenvManager(self.topsrcdir,
|
|
self.topobjdir,
|
|
self.virtualenv_root,
|
|
self.log_handle,
|
|
submanifest)
|
|
if not submanager.up_to_date():
|
|
return False
|
|
|
|
return True
|
|
|
|
def ensure(self):
|
|
"""Ensure the virtualenv is present and up to date.
|
|
|
|
If the virtualenv is up to date, this does nothing. Otherwise, it
|
|
creates and populates the virtualenv as necessary.
|
|
|
|
This should be the main API used from this class as it is the
|
|
highest-level.
|
|
"""
|
|
if self.up_to_date():
|
|
return self.virtualenv_root
|
|
return self.build()
|
|
|
|
def create(self):
|
|
"""Create a new, empty virtualenv.
|
|
|
|
Receives the path to virtualenv's virtualenv.py script (which will be
|
|
called out to), the path to create the virtualenv in, and a handle to
|
|
write output to.
|
|
"""
|
|
env = dict(os.environ)
|
|
env.pop('PYTHONDONTWRITEBYTECODE', None)
|
|
|
|
args = [sys.executable, self.virtualenv_script_path,
|
|
'--system-site-packages', self.virtualenv_root]
|
|
|
|
result = subprocess.call(args, stdout=self.log_handle,
|
|
stderr=subprocess.STDOUT, env=env)
|
|
|
|
if result:
|
|
raise Exception('Error creating virtualenv.')
|
|
|
|
return self.virtualenv_root
|
|
|
|
def packages(self):
|
|
with file(self.manifest_path, 'rU') as fh:
|
|
packages = [line.rstrip().split(':')
|
|
for line in fh]
|
|
return packages
|
|
|
|
def populate(self):
|
|
"""Populate the virtualenv.
|
|
|
|
The manifest file consists of colon-delimited fields. The first field
|
|
specifies the action. The remaining fields are arguments to that
|
|
action. The following actions are supported:
|
|
|
|
setup.py -- Invoke setup.py for a package. Expects the arguments:
|
|
1. relative path directory containing setup.py.
|
|
2. argument(s) to setup.py. e.g. "develop". Each program argument
|
|
is delimited by a colon. Arguments with colons are not yet
|
|
supported.
|
|
|
|
filename.pth -- Adds the path given as argument to filename.pth under
|
|
the virtualenv site packages directory.
|
|
|
|
optional -- This denotes the action as optional. The requested action
|
|
is attempted. If it fails, we issue a warning and go on. The
|
|
initial "optional" field is stripped then the remaining line is
|
|
processed like normal. e.g.
|
|
"optional:setup.py:python/foo:built_ext:-i"
|
|
|
|
copy -- Copies the given file in the virtualenv site packages
|
|
directory.
|
|
|
|
packages.txt -- Denotes that the specified path is a child manifest. It
|
|
will be read and processed as if its contents were concatenated
|
|
into the manifest being read.
|
|
|
|
objdir -- Denotes a relative path in the object directory to add to the
|
|
search path. e.g. "objdir:build" will add $topobjdir/build to the
|
|
search path.
|
|
|
|
Note that the Python interpreter running this function should be the
|
|
one from the virtualenv. If it is the system Python or if the
|
|
environment is not configured properly, packages could be installed
|
|
into the wrong place. This is how virtualenv's work.
|
|
"""
|
|
|
|
packages = self.packages()
|
|
|
|
def handle_package(package):
|
|
python_lib = distutils.sysconfig.get_python_lib()
|
|
if package[0] == 'setup.py':
|
|
assert len(package) >= 2
|
|
|
|
self.call_setup(os.path.join(self.topsrcdir, package[1]),
|
|
package[2:])
|
|
|
|
return True
|
|
|
|
if package[0] == 'copy':
|
|
assert len(package) == 2
|
|
|
|
src = os.path.join(self.topsrcdir, package[1])
|
|
dst = os.path.join(python_lib, os.path.basename(package[1]))
|
|
|
|
shutil.copy(src, dst)
|
|
|
|
return True
|
|
|
|
if package[0] == 'packages.txt':
|
|
assert len(package) == 2
|
|
|
|
src = os.path.join(self.topsrcdir, package[1])
|
|
assert os.path.isfile(src), "'%s' does not exist" % src
|
|
submanager = VirtualenvManager(self.topsrcdir,
|
|
self.topobjdir,
|
|
self.virtualenv_root,
|
|
self.log_handle,
|
|
src)
|
|
submanager.populate()
|
|
|
|
return True
|
|
|
|
if package[0].endswith('.pth'):
|
|
assert len(package) == 2
|
|
|
|
path = os.path.join(self.topsrcdir, package[1])
|
|
|
|
with open(os.path.join(python_lib, package[0]), 'a') as f:
|
|
# This path is relative to the .pth file. Using a
|
|
# relative path allows the srcdir/objdir combination
|
|
# to be moved around (as long as the paths relative to
|
|
# each other remain the same).
|
|
f.write("%s\n" % os.path.relpath(path, python_lib))
|
|
|
|
return True
|
|
|
|
if package[0] == 'optional':
|
|
try:
|
|
handle_package(package[1:])
|
|
return True
|
|
except:
|
|
print('Error processing command. Ignoring', \
|
|
'because optional. (%s)' % ':'.join(package),
|
|
file=self.log_handle)
|
|
return False
|
|
|
|
if package[0] == 'objdir':
|
|
assert len(package) == 2
|
|
path = os.path.join(self.topobjdir, package[1])
|
|
|
|
with open(os.path.join(python_lib, 'objdir.pth'), 'a') as f:
|
|
f.write('%s\n' % path)
|
|
|
|
return True
|
|
|
|
raise Exception('Unknown action: %s' % package[0])
|
|
|
|
# We always target the OS X deployment target that Python itself was
|
|
# built with, regardless of what's in the current environment. If we
|
|
# don't do # this, we may run into a Python bug. See
|
|
# http://bugs.python.org/issue9516 and bug 659881.
|
|
#
|
|
# Note that this assumes that nothing compiled in the virtualenv is
|
|
# shipped as part of a distribution. If we do ship anything, the
|
|
# deployment target here may be different from what's targeted by the
|
|
# shipping binaries and # virtualenv-produced binaries may fail to
|
|
# work.
|
|
#
|
|
# We also ignore environment variables that may have been altered by
|
|
# configure or a mozconfig activated in the current shell. We trust
|
|
# Python is smart enough to find a proper compiler and to use the
|
|
# proper compiler flags. If it isn't your Python is likely broken.
|
|
IGNORE_ENV_VARIABLES = ('CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS',
|
|
'PYTHONDONTWRITEBYTECODE')
|
|
|
|
try:
|
|
old_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', None)
|
|
sysconfig_target = \
|
|
distutils.sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET')
|
|
|
|
if sysconfig_target is not None:
|
|
os.environ['MACOSX_DEPLOYMENT_TARGET'] = sysconfig_target
|
|
|
|
old_env_variables = {}
|
|
for k in IGNORE_ENV_VARIABLES:
|
|
if k not in os.environ:
|
|
continue
|
|
|
|
old_env_variables[k] = os.environ[k]
|
|
del os.environ[k]
|
|
|
|
for package in packages:
|
|
handle_package(package)
|
|
finally:
|
|
os.environ.pop('MACOSX_DEPLOYMENT_TARGET', None)
|
|
|
|
if old_target is not None:
|
|
os.environ['MACOSX_DEPLOYMENT_TARGET'] = old_target
|
|
|
|
os.environ.update(old_env_variables)
|
|
|
|
def call_setup(self, directory, arguments):
|
|
"""Calls setup.py in a directory."""
|
|
setup = os.path.join(directory, 'setup.py')
|
|
|
|
program = [sys.executable, setup]
|
|
program.extend(arguments)
|
|
|
|
# We probably could call the contents of this file inside the context
|
|
# of this interpreter using execfile() or similar. However, if global
|
|
# variables like sys.path are adjusted, this could cause all kinds of
|
|
# havoc. While this may work, invoking a new process is safer.
|
|
|
|
try:
|
|
output = subprocess.check_output(program, cwd=directory, stderr=subprocess.STDOUT)
|
|
print(output)
|
|
except subprocess.CalledProcessError as e:
|
|
if 'Python.h: No such file or directory' in e.output:
|
|
print('WARNING: Python.h not found. Install Python development headers.')
|
|
else:
|
|
print(e.output)
|
|
|
|
raise Exception('Error installing package: %s' % directory)
|
|
|
|
def build(self):
|
|
"""Build a virtualenv per tree conventions.
|
|
|
|
This returns the path of the created virtualenv.
|
|
"""
|
|
|
|
self.create()
|
|
|
|
# We need to populate the virtualenv using the Python executable in
|
|
# the virtualenv for paths to be proper.
|
|
|
|
args = [self.python_path, __file__, 'populate', self.topsrcdir,
|
|
self.topobjdir, self.virtualenv_root]
|
|
|
|
result = subprocess.call(args, stdout=self.log_handle,
|
|
stderr=subprocess.STDOUT, cwd=self.topsrcdir)
|
|
|
|
if result != 0:
|
|
raise Exception('Error populating virtualenv.')
|
|
|
|
os.utime(self.activate_path, None)
|
|
|
|
return self.virtualenv_root
|
|
|
|
def activate(self):
|
|
"""Activate the virtualenv in this Python context.
|
|
|
|
If you run a random Python script and wish to "activate" the
|
|
virtualenv, you can simply instantiate an instance of this class
|
|
and call .ensure() and .activate() to make the virtualenv active.
|
|
"""
|
|
|
|
execfile(self.activate_path, dict(__file__=self.activate_path))
|
|
|
|
|
|
def verify_python_version(log_handle):
|
|
"""Ensure the current version of Python is sufficient."""
|
|
major, minor = sys.version_info[:2]
|
|
|
|
if major != MINIMUM_PYTHON_MAJOR or minor < MINIMUM_PYTHON_MINOR:
|
|
log_handle.write('Python %d.%d or greater (but not Python 3) is '
|
|
'required to build. ' %
|
|
(MINIMUM_PYTHON_MAJOR, MINIMUM_PYTHON_MINOR))
|
|
log_handle.write('You are running Python %d.%d.\n' % (major, minor))
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
if len(sys.argv) < 4:
|
|
print('Usage: populate_virtualenv.py /path/to/topsrcdir /path/to/topobjdir /path/to/virtualenv')
|
|
sys.exit(1)
|
|
|
|
verify_python_version(sys.stdout)
|
|
|
|
topsrcdir = sys.argv[1]
|
|
topobjdir = sys.argv[2]
|
|
virtualenv_path = sys.argv[3]
|
|
populate = False
|
|
|
|
# This should only be called internally.
|
|
if sys.argv[1] == 'populate':
|
|
populate = True
|
|
topsrcdir = sys.argv[2]
|
|
topobjdir = sys.argv[3]
|
|
virtualenv_path = sys.argv[4]
|
|
|
|
# path to default packages.txt
|
|
manifest_path = os.path.join(topsrcdir, 'build', 'virtualenv', 'packages.txt')
|
|
|
|
manager = VirtualenvManager(topsrcdir, topobjdir, virtualenv_path,
|
|
sys.stdout, manifest_path)
|
|
|
|
if populate:
|
|
manager.populate()
|
|
else:
|
|
manager.ensure()
|
|
|