Bug 1253502 - Move python virtualenv initialization to moz.configure. r=gps

aclocal.m4

@@ -26,7 +26,6 @@ builtin(include, build/autoconf/arch.m4)dnl
 builtin(include, build/autoconf/android.m4)dnl
 builtin(include, build/autoconf/zlib.m4)dnl
 builtin(include, build/autoconf/linux.m4)dnl
-builtin(include, build/autoconf/python-virtualenv.m4)dnl
 builtin(include, build/autoconf/winsdk.m4)dnl
 builtin(include, build/autoconf/icu.m4)dnl
 builtin(include, build/autoconf/ffi.m4)dnl

build/autoconf/python-virtualenv.m4

@@ -1,85 +0,0 @@
-dnl This Source Code Form is subject to the terms of the Mozilla Public
-dnl License, v. 2.0. If a copy of the MPL was not distributed with this
-dnl file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-AC_DEFUN([MOZ_PYTHON],
-[
-
-dnl We honor the Python path defined in an environment variable. This is used
-dnl to pass the virtualenv's Python from the main configure to SpiderMonkey's
-dnl configure, for example.
-if test -z "$PYTHON"; then
-  MOZ_PATH_PROGS(PYTHON, $PYTHON python2.7 python)
-  if test -z "$PYTHON"; then
-      AC_MSG_ERROR([python was not found in \$PATH])
-  fi
-else
-  AC_MSG_RESULT([Using Python from environment variable \$PYTHON])
-fi
-
-_virtualenv_topsrcdir=
-_virtualenv_populate_path=
-
-dnl If this is a mozilla-central, we'll find the virtualenv in the top
-dnl source directory. If this is a SpiderMonkey build, we assume we're at
-dnl js/src and try to find the virtualenv from the mozilla-central root.
-for base in $MOZILLA_CENTRAL_PATH $_topsrcdir $_topsrcdir/../..; do
-  possible=$base/python/mozbuild/mozbuild/virtualenv.py
-
-  if test -e $possible; then
-    _virtualenv_topsrcdir=$base
-    _virtualenv_populate_path=$possible
-    break
-  fi
-done
-
-if test -z $_virtualenv_populate_path; then
-    AC_MSG_ERROR([Unable to find Virtualenv population script. In order
-to build, you will need mozilla-central's virtualenv.
-
-If you are building from a mozilla-central checkout, you should never see this
-message. If you are building from a source archive, the source archive was
-likely not created properly (it is missing the virtualenv files).
-
-If you have a copy of mozilla-central available, define the
-MOZILLA_CENTRAL_PATH environment variable to the top source directory of
-mozilla-central and relaunch configure.])
-
-fi
-
-if test -z $DONT_POPULATE_VIRTUALENV; then
-  AC_MSG_RESULT([Creating Python environment])
-  dnl This verifies our Python version is sane and ensures the Python
-  dnl virtualenv is present and up to date. It sanitizes the environment
-  dnl for us, so we don't need to clean anything out.
-  $PYTHON $_virtualenv_populate_path \
-    $_virtualenv_topsrcdir $MOZ_BUILD_ROOT $MOZ_BUILD_ROOT/_virtualenv \
-    $_virtualenv_topsrcdir/build/virtualenv_packages.txt || exit 1
-
-  case "$host_os" in
-  mingw*)
-    PYTHON=`cd $MOZ_BUILD_ROOT && pwd -W`/_virtualenv/Scripts/python.exe
-    ;;
-  *)
-    PYTHON=$MOZ_BUILD_ROOT/_virtualenv/bin/python
-    ;;
-  esac
-fi
-
-AC_SUBST(PYTHON)
-
-AC_MSG_CHECKING([Python environment is Mozilla virtualenv])
-$PYTHON -c "import mozbuild.base"
-if test "$?" != 0; then
-    AC_MSG_ERROR([Python environment does not appear to be sane.])
-fi
-AC_MSG_RESULT([yes])
-
-PYTHON_SITE_PACKAGES=`$PYTHON -c "import distutils.sysconfig; print distutils.sysconfig.get_python_lib()"`
-if test -z "$PYTHON_SITE_PACKAGES"; then
-    AC_MSG_ERROR([Could not determine python site packages directory.])
-fi
-AC_SUBST([PYTHON_SITE_PACKAGES])
-
-])
-

build/moz.configure/init.configure

@@ -88,6 +88,84 @@ def mozconfig(current_project, mozconfig,  build_env, help):
     return mozconfig
 
 
+option(env='PYTHON', nargs=1, help='Python interpreter')
+
+# Setup python virtualenv
+# ==============================================================
+@depends('PYTHON', check_build_environment, mozconfig)
+@advanced
+def virtualenv_python(env_python, build_env, mozconfig):
+    import os
+    import sys
+    import subprocess
+    from mozbuild.virtualenv import (
+	VirtualenvManager,
+        verify_python_version,
+    )
+
+    python = env_python[0] if env_python else None
+
+    # Ideally we'd rely on the mozconfig injection from mozconfig_options,
+    # but we'd rather avoid the verbosity when we need to reexecute with
+    # a different python.
+    if mozconfig['path']:
+        if 'PYTHON' in mozconfig['env']['added']:
+            python = mozconfig['env']['added']['PYTHON']
+        elif 'PYTHON' in mozconfig['env']['modified']:
+            python = mozconfig['env']['modified']['PYTHON'][1]
+        elif 'PYTHON' in mozconfig['vars']['added']:
+            python = mozconfig['vars']['added']['PYTHON']
+        elif 'PYTHON' in mozconfig['vars']['modified']:
+            python = mozconfig['vars']['modified']['PYTHON'][1]
+
+    verify_python_version(sys.stderr)
+    topsrcdir, topobjdir = build_env['TOPSRCDIR'], build_env['TOPOBJDIR']
+    if topobjdir.endswith('/js/src'):
+        topobjdir = topobjdir[:-7]
+
+    manager = VirtualenvManager(
+        topsrcdir, topobjdir,
+        os.path.join(topobjdir, '_virtualenv'), sys.stdout,
+        os.path.join(topsrcdir, 'build', 'virtualenv_packages.txt'))
+
+    if python:
+        # If we're not in the virtualenv, we need the which module for
+        # find_program.
+        if normsep(sys.executable) != normsep(manager.python_path):
+            sys.path.append(os.path.join(topsrcdir, 'python', 'which'))
+        found_python = find_program(python)
+        if not found_python:
+            error('The PYTHON environment variable does not contain '
+                  'a valid path. Cannot find %s' % python)
+        python = found_python
+    else:
+        python = sys.executable
+
+    if not manager.up_to_date(python):
+        warn('Creating Python environment')
+        manager.build(python)
+
+    python = normsep(manager.python_path)
+
+    if python != normsep(sys.executable):
+        warn('Reexecuting in the virtualenv')
+        if env_python:
+            del os.environ['PYTHON']
+        # One would prefer to use os.execl, but that's completely borked on
+        # Windows.
+        sys.exit(subprocess.call([python] + sys.argv))
+
+    # We are now in the virtualenv
+    import distutils.sysconfig
+    if not distutils.sysconfig.get_python_lib():
+        error('Could not determine python site packages directory')
+
+    set_config('PYTHON', python)
+    return python
+
+
+# Inject mozconfig options
+# ==============================================================
 @template
 @advanced
 def command_line_helper():

build/moz.configure/old.configure

@@ -59,9 +59,11 @@ def autoconf(mozconfig, autoconf):
 
 option(env='OLD_CONFIGURE', nargs=1, help='Path to the old configure script')
 
-@depends('OLD_CONFIGURE', mozconfig, autoconf, check_build_environment, shell)
+@depends('OLD_CONFIGURE', mozconfig, autoconf, check_build_environment, shell,
+         virtualenv_python)
 @advanced
-def prepare_configure(old_configure, mozconfig, autoconf, build_env, shell):
+def prepare_configure(old_configure, mozconfig, autoconf, build_env, shell,
+                      python):
     import glob
     import itertools
     import subprocess
@@ -121,6 +123,8 @@ def prepare_configure(old_configure, mozconfig, autoconf, build_env, shell):
                 for key in mozconfig[t]['removed'].keys():
                     print("unset %s" % key, file=out)
 
+        print('PYTHON=%s' % quote(python), file=out)
+
     return cmd
 
 

configure.py

@@ -12,7 +12,6 @@ import sys
 
 
 base_dir = os.path.abspath(os.path.dirname(__file__))
-sys.path.append(os.path.join(base_dir, 'python', 'which'))
 sys.path.append(os.path.join(base_dir, 'python', 'mozbuild'))
 from mozbuild.configure import ConfigureSandbox
 

js/src/aclocal.m4

@@ -25,7 +25,6 @@ builtin(include, ../../build/autoconf/arch.m4)dnl
 builtin(include, ../../build/autoconf/android.m4)dnl
 builtin(include, ../../build/autoconf/zlib.m4)dnl
 builtin(include, ../../build/autoconf/linux.m4)dnl
-builtin(include, ../../build/autoconf/python-virtualenv.m4)dnl
 builtin(include, ../../build/autoconf/winsdk.m4)dnl
 builtin(include, ../../build/autoconf/icu.m4)dnl
 builtin(include, ../../build/autoconf/ffi.m4)dnl
@@ -45,3 +44,8 @@ __MOZ_AC_INIT_PREPARE($1)
 ])
 
 MOZ_PROG_CHECKMSYS()
+dnl This won't actually read the mozconfig, but data that configure.py
+dnl will have placed for us to read. Configure.py takes care of not reading
+dnl the mozconfig where appropriate but can still give us some variables
+dnl to read.
+MOZ_READ_MOZCONFIG(.)

js/src/old-configure.in

@@ -527,8 +527,6 @@ else
     AC_MSG_RESULT([yes])
 fi
 
-MOZ_PYTHON
-
 if test -z "$COMPILE_ENVIRONMENT"; then
     NSINSTALL_BIN='$(PYTHON) $(topsrcdir)/config/nsinstall.py'
 fi

old-configure.in

@@ -99,8 +99,6 @@ _objdir=`pwd -P`
 MOZ_BUILD_ROOT=`pwd -W 2>/dev/null || pwd -P`
 DIST="$MOZ_BUILD_ROOT/dist"
 
-MOZ_PYTHON
-
 MOZ_DEFAULT_COMPILER
 
 COMPILE_ENVIRONMENT=1
@@ -9106,8 +9104,6 @@ export MOZ_ZLIB_CFLAGS
 export MOZ_ZLIB_LIBS
 export MOZ_APP_NAME
 export MOZ_APP_REMOTINGNAME
-export DONT_POPULATE_VIRTUALENV=1
-export PYTHON
 export RUSTC
 export MOZILLA_CENTRAL_PATH=$_topsrcdir
 export STLPORT_CPPFLAGS