Bug 1473498 - [mach] Support running mach commands with python 3 r=glandium,mars

Credit: mars for making the shell POSIX compliant

This embeds a blacklist of every mach command that needs to run with Python 2
directly in the mach driver itself. Initially this is every mach command. We
then use a bit of shell to determine whether the command being run needs Python
2 or 3.

While this approach may seem a bit hacky, it has several benefits:

1. No need to add complex machinery in mach's registration code.
2. No need to spawn two separate Python interpreters in the event a different
   Python from the original interpreter is needed.
3. Perf impact is negligible.
4. New commands are Python 3 by default.

It is also only a temporary hack. Once all commands are running with Python 3,
we can revert back to the original mach driver.

Differential Revision: https://phabricator.services.mozilla.com/D36103

--HG--
extra : moz-landing-system : lando

build/mach_bootstrap.py

@@ -175,11 +175,12 @@ def bootstrap(topsrcdir, mozilla_dir=None):
     if mozilla_dir is None:
         mozilla_dir = topsrcdir
 
-    # Ensure we are running Python 2.7+. We put this check here so we generate a
-    # user-friendly error message rather than a cryptic stack trace on module
-    # import.
-    if sys.version_info[0] != 2 or sys.version_info[1] < 7:
-        print('Python 2.7 or above (but not Python 3) is required to run mach.')
+    # Ensure we are running Python 2.7 or 3.5+. We put this check here so we
+    # generate a user-friendly error message rather than a cryptic stack trace
+    # on module import.
+    major, minor = sys.version_info[:2]
+    if (major == 2 and minor < 7) or (major == 3 and minor < 5):
+        print('Python 2.7 or Python 3.5+ is required to run mach.')
         print('You are running Python', platform.python_version())
         sys.exit(1)
 

mach

@@ -3,13 +3,184 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# The beginning of this script is both valid shell and valid python,
+# The beginning of this script is both valid POSIX shell and valid Python,
 # such that the script starts with the shell and is reexecuted with
-# the right python.
-'''which' python2.7 > /dev/null && exec python2.7 "$0" "$@" || exec python "$0" "$@"
+# the right Python.
+
+# Embeds a shell script inside a Python triple quote. This pattern is valid
+# shell because `''':'`, `':'` and `:` are all equivalent, and `:` is a no-op.
+''':'
+py2commands="
+    addtest
+    analyze
+    android
+    android-emulator
+    artifact
+    awsy-test
+    bootstrap
+    browsertime
+    build
+    build-backend
+    buildsymbols
+    busted
+    cargo
+    check-spidermonkey
+    clang-format
+    clobber
+    compare-locales
+    compileflags
+    configure
+    cppunittest
+    cramtest
+    crashtest
+    devtools-css-db
+    doc
+    doctor
+    dxr
+    empty-makefiles
+    environment
+    eslint
+    file-info
+    firefox-ui-functional
+    fluent-migration-test
+    geckodriver
+    geckodriver-test
+    geckoview-junit
+    google
+    gradle
+    gtest
+    ide
+    import-pr
+    install
+    install-android
+    install-desktop
+    jsapi-tests
+    jsshell-bench
+    jstestbrowser
+    jstests
+    lint
+    mach-commands
+    mach-completion
+    mach-debug-commands
+    marionette-test
+    mdn
+    mochitest
+    mozbuild-reference
+    mozharness
+    mozregression
+    package
+    package-multi-locale
+    pastebin
+    power
+    prettier-format
+    puppeteer-test
+    python
+    python-safety
+    python-test
+    raptor
+    raptor-test
+    reftest
+    release
+    release-history
+    remote
+    repackage
+    resource-usage
+    robocop
+    run
+    run-android
+    run-desktop
+    rusttests
+    search
+    searchfox
+    settings
+    show-log
+    static-analysis
+    talos-test
+    taskcluster-build-image
+    taskcluster-load-image
+    taskgraph
+    telemetry-tests-client
+    test
+    test-info
+    tps-build
+    try
+    uuid
+    valgrind-test
+    vcs-setup
+    vendor
+    visualmetrics
+    warnings-list
+    warnings-summary
+    watch
+    web-platform-tests
+    web-platform-tests-update
+    webidl-example
+    webidl-parser-test
+    webrtc-gtest
+    wpt
+    wpt-manifest-update
+    wpt-metadata-merge
+    wpt-metadata-summary
+    wpt-serve
+    wpt-unittest
+    wpt-update
+    xpcshell-test
+"
+
+run_py() {
+    # Try to run a specific Python interpreter. Fall back to the system
+    # default Python if the specific interpreter couldn't be found.
+    py_executable="$1"
+    shift
+    if which "$py_executable" > /dev/null
+    then
+        exec "$py_executable" "$0" "$@"
+    elif [ "$py_executable" = "python2.7" ]; then
+        exec python "$0" "$@"
+    else
+        echo "This mach command requires $py_executable, which wasn't found on the system!"
+        exit 1
+    fi
+}
+
+first_arg=$1
+if [ -z "$first_arg" ]; then
+    run_py python3
+fi
+
+case "${first_arg}" in
+    "-"*)
+        # We have global arguments which are tricky to parse from this shell
+        # script. So invoke `mach` with a special --print-command argument to
+        # return the name of the command. This adds extra overhead when using
+        # global arguments, but global arguments are an edge case and this hack
+        # is only needed temporarily for the Python 3 migration. We use Python
+        # 2.7 because using Python 3 hits this error in build tasks:
+        # https://searchfox.org/mozilla-central/rev/c7e8bc4996f9/build/moz.configure/init.configure#319
+        command=`run_py python2.7 --print-command "$@" | tail -n1`
+        ;;
+    *)
+        # In the common case, the first argument is the command.
+        command=${first_arg};
+        ;;
+esac
+
+# Check for the mach subcommand in the Python 2 commands list and run it
+# with the correct interpreter.
+case " $(echo $py2commands) " in
+    *\ $command\ *)
+        run_py python2.7 "$@"
+        ;;
+    *)
+        run_py python3 "$@"
+        ;;
+esac
+
+# Run Python 3 for everything else.
+run_py python3 "$@"
 '''
 
-from __future__ import print_function, unicode_literals
+from __future__ import absolute_import, print_function, unicode_literals
 
 import os
 import sys
@@ -22,11 +193,15 @@ def ancestors(path):
             break
 
 def load_mach(dir_path, mach_path):
-    import imp
-    with open(mach_path, 'r') as fh:
-        imp.load_module('mach_bootstrap', fh, mach_path,
-                        ('.py', 'r', imp.PY_SOURCE))
-    import mach_bootstrap
+    if sys.version_info < (3, 5):
+        import imp
+        mach_bootstrap = imp.load_source('mach_bootstrap', mach_path)
+    else:
+        import importlib.util
+        spec = importlib.util.spec_from_file_location('mach_bootstrap', mach_path)
+        mach_bootstrap = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(mach_bootstrap)
+
     return mach_bootstrap.bootstrap(dir_path)
 
 

python/mach/mach/main.py

@@ -429,6 +429,17 @@ To see more help for a specific command, run:
                                                  ' '.join(e.arguments)))
             return 1
 
+        if not hasattr(args, 'mach_handler'):
+            raise MachError('ArgumentParser result missing mach handler info.')
+
+        handler = getattr(args, 'mach_handler')
+
+        # This is used by the `mach` driver to find the command name amidst
+        # global arguments.
+        if args.print_command:
+            print(handler.name)
+            sys.exit(0)
+
         # Add JSON logging to a file if requested.
         if args.logfile:
             self.log_manager.add_json_handler(args.logfile)
@@ -455,11 +466,6 @@ To see more help for a specific command, run:
             # to command line handling (e.g alias, defaults) will be ignored.
             self.load_settings(args.settings_file)
 
-        if not hasattr(args, 'mach_handler'):
-            raise MachError('ArgumentParser result missing mach handler info.')
-
-        handler = getattr(args, 'mach_handler')
-
         # if --disable-tests flag was enabled in the mozconfig used to compile
         # the build, tests will be disabled.
         # instead of trying to run nonexistent tests then reporting a failure,
@@ -623,6 +629,8 @@ To see more help for a specific command, run:
         global_group.add_argument('--settings', dest='settings_file',
                                   metavar='FILENAME', default=None,
                                   help='Path to settings file.')
+        global_group.add_argument('--print-command', action='store_true',
+                                  help=argparse.SUPPRESS)
 
         for args, kwargs in self.global_arguments:
             global_group.add_argument(*args, **kwargs)

python/mach/mach/registrar.py

@@ -2,11 +2,13 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from __future__ import print_function
-from __future__ import absolute_import, unicode_literals
+from __future__ import absolute_import, print_function, unicode_literals
+
+import time
+
+import six
 
 from .base import MachError
-import time
 
 INVALID_COMMAND_CONTEXT = r'''
 It looks like you tried to run a mach command from an invalid context. The %s
@@ -133,7 +135,7 @@ class MachRegistrar(object):
         end_time = time.time()
 
         result = result or 0
-        assert isinstance(result, (int, long))
+        assert isinstance(result, six.integer_types)
 
         if context and not debug_command:
             postrun = getattr(context, 'post_dispatch_handler', None)