Bug 1725895: Port --profile-command to pure-Python r=nalexander,glandium

As part of this, the shell-script part of `./mach` can be removed,
making it pure Python.

There's a change in `--profile-command` behaviour, though: it now only
profiles the specific command, rather than all of Mach.
This is because _so much of Mach_ has already been run before
CLI arguments are parsed in the Python process.

If a developer wants to profile Mach itself, they can manually run
`python3 -m cProfile -o <file> ./mach ...`

Differential Revision: https://phabricator.services.mozilla.com/D133928
This commit is contained in:
Mitchell Hentges 2022-01-06 00:32:48 +00:00
parent becd98a1bb
commit 7e6a4952a2
7 changed files with 52 additions and 101 deletions

View File

@ -116,17 +116,17 @@ def bootstrap_path(path, **kwargs):
"--enable-bootstrap",
toolchains_base_dir,
bootstrap_toolchain_tasks,
shell,
build_environment,
dependable(path),
when=when,
)
@imports("os")
@imports("subprocess")
@imports("sys")
@imports(_from="mozbuild.util", _import="ensureParentDir")
@imports(_from="__builtin__", _import="open")
@imports(_from="__builtin__", _import="Exception")
def bootstrap_path(bootstrap, toolchains_base_dir, tasks, shell, build_env, path):
def bootstrap_path(bootstrap, toolchains_base_dir, tasks, build_env, path):
path_parts = path.split("/")
def try_bootstrap(exists):
@ -169,7 +169,7 @@ def bootstrap_path(path, **kwargs):
os.makedirs(toolchains_base_dir, exist_ok=True)
subprocess.run(
[
shell,
sys.executable,
os.path.join(build_env.topsrcdir, "mach"),
"--log-no-times",
"artifact",

View File

@ -470,7 +470,7 @@ mach = posixpath.join(PDIR.source, "mach")
if not args.nobuild:
# Do the build
run_command([mach, "build"], check=True)
run_command([sys.executable, mach, "build"], check=True)
if use_minidump:
# Convert symbols to breakpad format.
@ -481,6 +481,7 @@ if not args.nobuild:
cmd_env["MOZ_AUTOMATION_BUILD_SYMBOLS"] = "1"
run_command(
[
sys.executable,
mach,
"build",
"recurse_syms",

76
mach
View File

@ -1,82 +1,8 @@
#!/bin/sh
#!/usr/bin/env python3
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# The beginning of this script is both valid POSIX shell and valid Python,
# such that the script starts with the shell and is reexecuted with
# the right Python.
# Embeds a shell script inside a Python triple quote. This pattern is valid
# shell because `''':'`, `':'` and `:` are all equivalent, and `:` is a no-op.
''':'
get_command() {
# Parse the name of the mach command out of the arguments. This is necessary
# in the presence of global mach arguments that come before the name of the
# command, e.g. `mach -v build`. We dispatch to the correct Python
# interpreter depending on the command.
while true; do
case $1 in
-v|--verbose) shift;;
-l|--log-file)
if [ "$#" -lt 2 ]
then
echo
break
else
shift 2
fi
;;
--no-interactive) shift;;
--log-interval) shift;;
--log-no-times) shift;;
-h) shift;;
--debug-command) shift;;
--profile-command)
py_profile_command="1"
shift;;
--settings)
if [ "$#" -lt 2 ]
then
echo
break
else
shift 2
fi
;;
"") echo; break;;
*) echo $1; break;;
esac
done
return ${py_profile_command}
}
command=$(get_command "$@")
py_profile_command=$?
if [ ${py_profile_command} -eq 0 ]
then
py_profile_command_args=""
else
# We would prefer to use an array variable here, but we're limited to POSIX.
# None of our arguments have quoting or spaces so we can safely interpolate
# a string instead.
py_profile_command_args="-m cProfile -o mach_profile_${command}.cProfile"
echo "Running with --profile-command. To visualize, use snakeviz:"
echo "python3 -m pip install snakeviz"
echo "python3 -m snakeviz mach_profile_${command}.cProfile"
fi
if command -v python3 > /dev/null
then
exec python3 $py_profile_command_args "$0" "$@"
else
echo "This mach command requires 'python3', which wasn't found on the system!"
exit 1
fi
'''
from __future__ import absolute_import, print_function, unicode_literals
import os

View File

@ -33,18 +33,31 @@ when the command is invoked with:
How do I profile a slow command?
--------------------------------
You can run a command and capture a profile as the ``mach`` process
loads and invokes the command with:
To diagnose bottlenecks, you can collect a performance profile:
.. code-block:: shell
./mach --profile-command SLOW-COMMAND ARGS ...
./mach --profile-command SLOW-COMMAND ARGS ...
Look for a ``mach_profile_SLOW-COMMAND.cProfile`` file. You can
visualize using `snakeviz <https://jiffyclub.github.io/snakeviz/>`__.
Instructions on how to install and use ``snakeviz`` are printed to the
console, since it can be tricky to target the correct Python virtual
environment.
Then, you can visualize ``mach_profile_SLOW-COMMAND.cProfile`` using
`snakeviz <https://jiffyclub.github.io/snakeviz/>`__:
.. code-block:: shell
# If you don't have snakeviz installed yet:
python3 -m pip install snakeviz
python3 -m snakeviz mach_profile_SLOW-COMMAND.cProfile
How do I profile ``mach`` itself?
---------------------------------
Since ``--profile-command`` only profiles commands, you'll need to invoke ``cProfile``
directly to profile ``mach`` itself:
.. code-block:: shell
python3 -m cProfile -o mach.cProfile ./mach ...
python3 -m snakeviz mach.cProfile
Is ``mach`` a build system?
---------------------------

View File

@ -490,6 +490,7 @@ To see more help for a specific command, run:
handler,
context,
debug_command=args.debug_command,
profile_command=args.profile_command,
**vars(args.command_args),
)
except KeyboardInterrupt as ki:

View File

@ -5,6 +5,8 @@
from __future__ import absolute_import, print_function, unicode_literals
import time
from cProfile import Profile
from pathlib import Path
import six
@ -85,7 +87,9 @@ class MachRegistrar(object):
return fail_conditions
def _run_command_handler(self, handler, context, debug_command=False, **kwargs):
def _run_command_handler(
self, handler, context, debug_command=False, profile_command=False, **kwargs
):
instance = MachRegistrar._instance(handler, context, **kwargs)
fail_conditions = MachRegistrar._fail_conditions(handler, instance)
if fail_conditions:
@ -97,6 +101,11 @@ class MachRegistrar(object):
self.command_depth += 1
fn = handler.func
profile = None
if profile_command:
profile = Profile()
profile.enable()
start_time = time.time()
if debug_command:
@ -108,6 +117,19 @@ class MachRegistrar(object):
end_time = time.time()
if profile_command:
profile.disable()
profile_file = (
Path(context.topdir) / f"mach_profile_{handler.name}.cProfile"
)
profile.dump_stats(profile_file)
print(
f'Mach command profile created at "{profile_file}". To visualize, use '
f"snakeviz:"
)
print("python3 -m pip install snakeviz")
print(f"python3 -m snakeviz {profile_file.name}")
result = result or 0
assert isinstance(result, six.integer_types)

View File

@ -812,19 +812,7 @@ items from that key's value."
)
def _query_mach(self):
dirs = self.query_abs_dirs()
if "MOZILLABUILD" in os.environ:
# We found many issues with intermittent build failures when not
# invoking mach via bash.
# See bug 1364651 before considering changing.
mach = [
os.path.join(os.environ["MOZILLABUILD"], "msys", "bin", "bash.exe"),
os.path.join(dirs["abs_src_dir"], "mach"),
]
else:
mach = [sys.executable, "mach"]
return mach
return [sys.executable, "mach"]
def _run_mach_command_in_build_env(self, args, use_subprocess=False):
"""Run a mach command in a build context."""