Bug 1725895: Port --profile-command to pure-Python r=nalexander,glandium

As part of this, the shell-script part of `./mach` can be removed,
making it pure Python.

There's a change in `--profile-command` behaviour, though: it now only
profiles the specific command, rather than all of Mach.
This is because _so much of Mach_ has already been run before
CLI arguments are parsed in the Python process.

If a developer wants to profile Mach itself, they can manually run
`python3 -m cProfile -o <file> ./mach ...`

Differential Revision: https://phabricator.services.mozilla.com/D133928

build/moz.configure/bootstrap.configure

@@ -116,17 +116,17 @@ def bootstrap_path(path, **kwargs):
         "--enable-bootstrap",
         toolchains_base_dir,
         bootstrap_toolchain_tasks,
-        shell,
         build_environment,
         dependable(path),
         when=when,
     )
     @imports("os")
     @imports("subprocess")
+    @imports("sys")
     @imports(_from="mozbuild.util", _import="ensureParentDir")
     @imports(_from="__builtin__", _import="open")
     @imports(_from="__builtin__", _import="Exception")
-    def bootstrap_path(bootstrap, toolchains_base_dir, tasks, shell, build_env, path):
+    def bootstrap_path(bootstrap, toolchains_base_dir, tasks, build_env, path):
         path_parts = path.split("/")
 
         def try_bootstrap(exists):
@@ -169,7 +169,7 @@ def bootstrap_path(path, **kwargs):
             os.makedirs(toolchains_base_dir, exist_ok=True)
             subprocess.run(
                 [
-                    shell,
+                    sys.executable,
                     os.path.join(build_env.topsrcdir, "mach"),
                     "--log-no-times",
                     "artifact",

js/src/devtools/automation/autospider.py

@@ -470,7 +470,7 @@ mach = posixpath.join(PDIR.source, "mach")
 
 if not args.nobuild:
     # Do the build
-    run_command([mach, "build"], check=True)
+    run_command([sys.executable, mach, "build"], check=True)
 
     if use_minidump:
         # Convert symbols to breakpad format.
@@ -481,6 +481,7 @@ if not args.nobuild:
         cmd_env["MOZ_AUTOMATION_BUILD_SYMBOLS"] = "1"
         run_command(
             [
+                sys.executable,
                 mach,
                 "build",
                 "recurse_syms",

mach

@@ -1,82 +1,8 @@
-#!/bin/sh
+#!/usr/bin/env python3
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# The beginning of this script is both valid POSIX shell and valid Python,
-# such that the script starts with the shell and is reexecuted with
-# the right Python.
-
-# Embeds a shell script inside a Python triple quote. This pattern is valid
-# shell because `''':'`, `':'` and `:` are all equivalent, and `:` is a no-op.
-''':'
-
-get_command() {
-    # Parse the name of the mach command out of the arguments. This is necessary
-    # in the presence of global mach arguments that come before the name of the
-    # command, e.g. `mach -v build`. We dispatch to the correct Python
-    # interpreter depending on the command.
-    while true; do
-    case $1 in
-        -v|--verbose) shift;;
-        -l|--log-file)
-            if [ "$#" -lt 2 ]
-            then
-                echo
-                break
-            else
-                shift 2
-            fi
-            ;;
-        --no-interactive) shift;;
-        --log-interval) shift;;
-        --log-no-times) shift;;
-        -h) shift;;
-        --debug-command) shift;;
-        --profile-command)
-            py_profile_command="1"
-            shift;;
-        --settings)
-            if [ "$#" -lt 2 ]
-            then
-                echo
-                break
-            else
-                shift 2
-            fi
-            ;;
-        "") echo; break;;
-        *) echo $1; break;;
-    esac
-    done
-    return ${py_profile_command}
-}
-
-command=$(get_command "$@")
-py_profile_command=$?
-
-if [ ${py_profile_command} -eq 0 ]
-then
-    py_profile_command_args=""
-else
-    # We would prefer to use an array variable here, but we're limited to POSIX.
-    # None of our arguments have quoting or spaces so we can safely interpolate
-    # a string instead.
-    py_profile_command_args="-m cProfile -o mach_profile_${command}.cProfile"
-    echo "Running with --profile-command.  To visualize, use snakeviz:"
-    echo "python3 -m pip install snakeviz"
-    echo "python3 -m snakeviz mach_profile_${command}.cProfile"
-fi
-
-if command -v python3 > /dev/null
-then
-    exec python3 $py_profile_command_args "$0" "$@"
-else
-    echo "This mach command requires 'python3', which wasn't found on the system!"
-    exit 1
-fi
-'''
-
 from __future__ import absolute_import, print_function, unicode_literals
 
 import os

python/mach/docs/faq.rst

@@ -33,18 +33,31 @@ when the command is invoked with:
 How do I profile a slow command?
 --------------------------------
 
-You can run a command and capture a profile as the ``mach`` process
-loads and invokes the command with:
+To diagnose bottlenecks, you can collect a performance profile:
 
 .. code-block:: shell
 
-   ./mach --profile-command SLOW-COMMAND ARGS ...
+    ./mach --profile-command SLOW-COMMAND ARGS ...
 
-Look for a ``mach_profile_SLOW-COMMAND.cProfile`` file.  You can
-visualize using `snakeviz <https://jiffyclub.github.io/snakeviz/>`__.
-Instructions on how to install and use ``snakeviz`` are printed to the
-console, since it can be tricky to target the correct Python virtual
-environment.
+Then, you can visualize ``mach_profile_SLOW-COMMAND.cProfile`` using
+`snakeviz <https://jiffyclub.github.io/snakeviz/>`__:
+
+.. code-block:: shell
+
+    # If you don't have snakeviz installed yet:
+    python3 -m pip install snakeviz
+    python3 -m snakeviz mach_profile_SLOW-COMMAND.cProfile
+
+How do I profile ``mach`` itself?
+---------------------------------
+
+Since ``--profile-command`` only profiles commands, you'll need to invoke ``cProfile``
+directly to profile ``mach`` itself:
+
+.. code-block:: shell
+
+    python3 -m cProfile -o mach.cProfile ./mach ...
+    python3 -m snakeviz mach.cProfile
 
 Is ``mach`` a build system?
 ---------------------------

python/mach/mach/main.py

@@ -490,6 +490,7 @@ To see more help for a specific command, run:
                 handler,
                 context,
                 debug_command=args.debug_command,
+                profile_command=args.profile_command,
                 **vars(args.command_args),
             )
         except KeyboardInterrupt as ki:

python/mach/mach/registrar.py

@@ -5,6 +5,8 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 import time
+from cProfile import Profile
+from pathlib import Path
 
 import six
 
@@ -85,7 +87,9 @@ class MachRegistrar(object):
 
         return fail_conditions
 
-    def _run_command_handler(self, handler, context, debug_command=False, **kwargs):
+    def _run_command_handler(
+        self, handler, context, debug_command=False, profile_command=False, **kwargs
+    ):
         instance = MachRegistrar._instance(handler, context, **kwargs)
         fail_conditions = MachRegistrar._fail_conditions(handler, instance)
         if fail_conditions:
@@ -97,6 +101,11 @@ class MachRegistrar(object):
         self.command_depth += 1
         fn = handler.func
 
+        profile = None
+        if profile_command:
+            profile = Profile()
+            profile.enable()
+
         start_time = time.time()
 
         if debug_command:
@@ -108,6 +117,19 @@ class MachRegistrar(object):
 
         end_time = time.time()
 
+        if profile_command:
+            profile.disable()
+            profile_file = (
+                Path(context.topdir) / f"mach_profile_{handler.name}.cProfile"
+            )
+            profile.dump_stats(profile_file)
+            print(
+                f'Mach command profile created at "{profile_file}". To visualize, use '
+                f"snakeviz:"
+            )
+            print("python3 -m pip install snakeviz")
+            print(f"python3 -m snakeviz {profile_file.name}")
+
         result = result or 0
         assert isinstance(result, six.integer_types)
 

testing/mozharness/mozharness/mozilla/building/buildbase.py

@@ -812,19 +812,7 @@ items from that key's value."
         )
 
     def _query_mach(self):
-        dirs = self.query_abs_dirs()
-
-        if "MOZILLABUILD" in os.environ:
-            # We found many issues with intermittent build failures when not
-            # invoking mach via bash.
-            # See bug 1364651 before considering changing.
-            mach = [
-                os.path.join(os.environ["MOZILLABUILD"], "msys", "bin", "bash.exe"),
-                os.path.join(dirs["abs_src_dir"], "mach"),
-            ]
-        else:
-            mach = [sys.executable, "mach"]
-        return mach
+        return [sys.executable, "mach"]
 
     def _run_mach_command_in_build_env(self, args, use_subprocess=False):
         """Run a mach command in a build context."""