Backed out 2 changesets (bug 1725895, bug 1740123) for causing build bustages.

Backed out changeset a6eabd95e31d (bug 1740123)
Backed out changeset bfd5211060b4 (bug 1725895)

build/mach_initialize.py

@@ -6,6 +6,7 @@ from __future__ import division, print_function, unicode_literals
 
 import math
 import os
+import platform
 import shutil
 import sys
 
@@ -139,6 +140,31 @@ CATEGORIES = {
     },
 }
 
+INSTALL_PYTHON_GUIDANCE_LINUX = """
+See https://firefox-source-docs.mozilla.org/setup/linux_build.html#installingpython
+for guidance on how to install Python on your system.
+""".strip()
+
+INSTALL_PYTHON_GUIDANCE_OSX = """
+See https://firefox-source-docs.mozilla.org/setup/macos_build.html
+for guidance on how to prepare your system to build Firefox. Perhaps
+you need to update Xcode, or install Python using brew?
+""".strip()
+
+INSTALL_PYTHON_GUIDANCE_MOZILLABUILD = """
+Python is provided by MozillaBuild; ensure your MozillaBuild
+installation is up to date.
+See https://firefox-source-docs.mozilla.org/setup/windows_build.html#install-mozillabuild
+for details.
+""".strip()
+
+INSTALL_PYTHON_GUIDANCE_OTHER = """
+We do not have specific instructions for your platform on how to
+install Python. You may find Pyenv (https://github.com/pyenv/pyenv)
+helpful, if your system package manager does not provide a way to
+install a recent enough Python 3.
+""".strip()
+
 
 def _activate_python_environment(topsrcdir, get_state_dir):
     from mach.site import MachSiteManager
@@ -151,6 +177,21 @@ def _activate_python_environment(topsrcdir, get_state_dir):
 
 
 def initialize(topsrcdir):
+    # Ensure we are running Python 3.6+. We run this check as soon as
+    # possible to avoid a cryptic import/usage error.
+    if sys.version_info < (3, 6):
+        print("Python 3.6+ is required to run mach.")
+        print("You are running Python", platform.python_version())
+        if sys.platform.startswith("linux"):
+            print(INSTALL_PYTHON_GUIDANCE_LINUX)
+        elif sys.platform.startswith("darwin"):
+            print(INSTALL_PYTHON_GUIDANCE_OSX)
+        elif "MOZILLABUILD" in os.environ:
+            print(INSTALL_PYTHON_GUIDANCE_MOZILLABUILD)
+        else:
+            print(INSTALL_PYTHON_GUIDANCE_OTHER)
+        sys.exit(1)
+
     # This directory was deleted in bug 1666345, but there may be some ignored
     # files here. We can safely just delete it for the user so they don't have
     # to clean the repo themselves.

build/moz.configure/bootstrap.configure

@@ -116,17 +116,17 @@ def bootstrap_path(path, **kwargs):
         "--enable-bootstrap",
         toolchains_base_dir,
         bootstrap_toolchain_tasks,
+        shell,
         build_environment,
         dependable(path),
         when=when,
     )
     @imports("os")
     @imports("subprocess")
-    @imports("sys")
     @imports(_from="mozbuild.util", _import="ensureParentDir")
     @imports(_from="__builtin__", _import="open")
     @imports(_from="__builtin__", _import="Exception")
-    def bootstrap_path(bootstrap, toolchains_base_dir, tasks, build_env, path):
+    def bootstrap_path(bootstrap, toolchains_base_dir, tasks, shell, build_env, path):
         path_parts = path.split("/")
 
         def try_bootstrap(exists):
@@ -169,7 +169,7 @@ def bootstrap_path(path, **kwargs):
             os.makedirs(toolchains_base_dir, exist_ok=True)
             subprocess.run(
                 [
-                    sys.executable,
+                    shell,
                     os.path.join(build_env.topsrcdir, "mach"),
                     "--log-no-times",
                     "artifact",

js/src/devtools/automation/autospider.py

@@ -470,7 +470,7 @@ mach = posixpath.join(PDIR.source, "mach")
 
 if not args.nobuild:
     # Do the build
-    run_command([sys.executable, mach, "build"], check=True)
+    run_command([mach, "build"], check=True)
 
     if use_minidump:
         # Convert symbols to breakpad format.
@@ -481,7 +481,6 @@ if not args.nobuild:
         cmd_env["MOZ_AUTOMATION_BUILD_SYMBOLS"] = "1"
         run_command(
             [
-                sys.executable,
                 mach,
                 "build",
                 "recurse_syms",

mach

@@ -1,19 +1,89 @@
-#!/usr/bin/env python3
+#!/bin/sh
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+# The beginning of this script is both valid POSIX shell and valid Python,
+# such that the script starts with the shell and is reexecuted with
+# the right Python.
+
+# Embeds a shell script inside a Python triple quote. This pattern is valid
+# shell because `''':'`, `':'` and `:` are all equivalent, and `:` is a no-op.
+''':'
+
+get_command() {
+    # Parse the name of the mach command out of the arguments. This is necessary
+    # in the presence of global mach arguments that come before the name of the
+    # command, e.g. `mach -v build`. We dispatch to the correct Python
+    # interpreter depending on the command.
+    while true; do
+    case $1 in
+        -v|--verbose) shift;;
+        -l|--log-file)
+            if [ "$#" -lt 2 ]
+            then
+                echo
+                break
+            else
+                shift 2
+            fi
+            ;;
+        --no-interactive) shift;;
+        --log-interval) shift;;
+        --log-no-times) shift;;
+        -h) shift;;
+        --debug-command) shift;;
+        --profile-command)
+            py_profile_command="1"
+            shift;;
+        --settings)
+            if [ "$#" -lt 2 ]
+            then
+                echo
+                break
+            else
+                shift 2
+            fi
+            ;;
+        "") echo; break;;
+        *) echo $1; break;;
+    esac
+    done
+    return ${py_profile_command}
+}
+
+command=$(get_command "$@")
+py_profile_command=$?
+
+if [ ${py_profile_command} -eq 0 ]
+then
+    py_profile_command_args=""
+else
+    # We would prefer to use an array variable here, but we're limited to POSIX.
+    # None of our arguments have quoting or spaces so we can safely interpolate
+    # a string instead.
+    py_profile_command_args="-m cProfile -o mach_profile_${command}.cProfile"
+    echo "Running with --profile-command.  To visualize, use snakeviz:"
+    echo "python3 -m pip install snakeviz"
+    echo "python3 -m snakeviz mach_profile_${command}.cProfile"
+fi
+
+if command -v python3 > /dev/null
+then
+    exec python3 $py_profile_command_args "$0" "$@"
+else
+    echo "This mach command requires 'python3', which wasn't found on the system!"
+    exit 1
+fi
+'''
+
 from __future__ import absolute_import, print_function, unicode_literals
 
 import os
-import platform
 import sys
-from textwrap import dedent
 
 
 def load_mach(dir_path, mach_path):
-    # Defer import of "importlib.util" until after Python version check has happened
-    # so that Python 2 usages fail gracefully.
     import importlib.util
     spec = importlib.util.spec_from_file_location('mach_initialize', mach_path)
     mach_initialize = importlib.util.module_from_spec(spec)
@@ -37,37 +107,6 @@ def check_and_get_mach(dir_path):
 
 
 def main(args):
-    # Ensure we are running Python 3.6+. We run this check as soon as
-    # possible to avoid a cryptic import/usage error.
-    if sys.version_info < (3, 6):
-        print("Python 3.6+ is required to run mach.")
-        print("You are running Python", platform.python_version())
-        if sys.platform.startswith("linux"):
-            print(dedent("""
-            See https://firefox-source-docs.mozilla.org/setup/linux_build.html#installingpython
-            for guidance on how to install Python on your system.
-            """).strip())
-        elif sys.platform.startswith("darwin"):
-            print(dedent("""
-            See https://firefox-source-docs.mozilla.org/setup/macos_build.html
-            for guidance on how to prepare your system to build Firefox. Perhaps
-            you need to update Xcode, or install Python using brew?
-            """).strip())
-        elif "MOZILLABUILD" in os.environ:
-            print(dedent("""
-            Python is provided by MozillaBuild; ensure your MozillaBuild installation is
-            up to date. See https://firefox-source-docs.mozilla.org/setup/windows_build.html#install-mozillabuild
-            for details.
-            """).strip())
-        else:
-            print(dedent("""
-            We do not have specific instructions for your platform on how to
-            install Python. You may find Pyenv (https://github.com/pyenv/pyenv)
-            helpful, if your system package manager does not provide a way to
-            install a recent enough Python 3.
-            """).strip())
-        sys.exit(1)
-
     # XCode python sets __PYVENV_LAUNCHER__, which overrides the executable
     # used when a python subprocess is created. This is an issue when we want
     # to run using our virtualenv python executables.

python/mach/docs/faq.rst

@@ -33,31 +33,18 @@ when the command is invoked with:
 How do I profile a slow command?
 --------------------------------
 
-To diagnose bottlenecks, you can collect a performance profile:
+You can run a command and capture a profile as the ``mach`` process
+loads and invokes the command with:
 
 .. code-block:: shell
 
-    ./mach --profile-command SLOW-COMMAND ARGS ...
+   ./mach --profile-command SLOW-COMMAND ARGS ...
 
-Then, you can visualize ``mach_profile_SLOW-COMMAND.cProfile`` using
-`snakeviz <https://jiffyclub.github.io/snakeviz/>`__:
-
-.. code-block:: shell
-
-    # If you don't have snakeviz installed yet:
-    python3 -m pip install snakeviz
-    python3 -m snakeviz mach_profile_SLOW-COMMAND.cProfile
-
-How do I profile ``mach`` itself?
----------------------------------
-
-Since ``--profile-command`` only profiles commands, you'll need to invoke ``cProfile``
-directly to profile ``mach`` itself:
-
-.. code-block:: shell
-
-    python3 -m cProfile -o mach.cProfile ./mach ...
-    python3 -m snakeviz mach.cProfile
+Look for a ``mach_profile_SLOW-COMMAND.cProfile`` file.  You can
+visualize using `snakeviz <https://jiffyclub.github.io/snakeviz/>`__.
+Instructions on how to install and use ``snakeviz`` are printed to the
+console, since it can be tricky to target the correct Python virtual
+environment.
 
 Is ``mach`` a build system?
 ---------------------------

python/mach/mach/main.py

@@ -490,7 +490,6 @@ To see more help for a specific command, run:
                 handler,
                 context,
                 debug_command=args.debug_command,
-                profile_command=args.profile_command,
                 **vars(args.command_args),
             )
         except KeyboardInterrupt as ki:

python/mach/mach/registrar.py

@@ -5,8 +5,6 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 import time
-from cProfile import Profile
-from pathlib import Path
 
 import six
 
@@ -87,9 +85,7 @@ class MachRegistrar(object):
 
         return fail_conditions
 
-    def _run_command_handler(
-        self, handler, context, debug_command=False, profile_command=False, **kwargs
-    ):
+    def _run_command_handler(self, handler, context, debug_command=False, **kwargs):
         instance = MachRegistrar._instance(handler, context, **kwargs)
         fail_conditions = MachRegistrar._fail_conditions(handler, instance)
         if fail_conditions:
@@ -101,11 +97,6 @@ class MachRegistrar(object):
         self.command_depth += 1
         fn = handler.func
 
-        profile = None
-        if profile_command:
-            profile = Profile()
-            profile.enable()
-
         start_time = time.time()
 
         if debug_command:
@@ -117,19 +108,6 @@ class MachRegistrar(object):
 
         end_time = time.time()
 
-        if profile_command:
-            profile.disable()
-            profile_file = (
-                Path(context.topdir) / f"mach_profile_{handler.name}.cProfile"
-            )
-            profile.dump_stats(profile_file)
-            print(
-                f'Mach command profile created at "{profile_file}". To visualize, use '
-                f"snakeviz:"
-            )
-            print("python3 -m pip install snakeviz")
-            print(f"python3 -m snakeviz {profile_file.name}")
-
         result = result or 0
         assert isinstance(result, six.integer_types)