Bug 1526072 - Record cpu utilization and clobber/full builds in build telemetry. r=nalexander

Differential Revision: https://phabricator.services.mozilla.com/D22630

--HG--
extra : moz-landing-system : lando
This commit is contained in:
Chris Manchester 2019-03-27 22:42:48 +00:00
parent 584f26283b
commit 5c005e476e
4 changed files with 36 additions and 3 deletions

View File

@ -250,6 +250,8 @@ def bootstrap(topsrcdir, mozilla_dir=None):
except Exception:
substs = {}
command_attrs = getattr(context, 'command_attrs', {})
# We gather telemetry for every operation.
paths = {
instance.topsrcdir: '$topsrcdir/',
@ -263,7 +265,7 @@ def bootstrap(topsrcdir, mozilla_dir=None):
data = gather_telemetry(command=handler.name, success=(result == 0),
start_time=start_time, end_time=end_time,
mach_context=context, substs=substs,
paths=paths)
command_attrs=command_attrs, paths=paths)
if data:
telemetry_dir = os.path.join(get_state_dir(), 'telemetry')
try:

View File

@ -12,6 +12,7 @@ class CommandContext(object):
self.settings = settings
self.log_manager = log_manager
self.commands = commands
self.command_attrs = {}
for k, v in kwargs.items():
setattr(self, k, v)

View File

@ -978,6 +978,10 @@ class CCacheStats(object):
class BuildDriver(MozbuildObject):
"""Provides a high-level API for build actions."""
def __init__(self, *args, **kwargs):
MozbuildObject.__init__(self, *args, **kwargs)
self.mach_context = None
def build(self, what=None, disable_extra_make_dependencies=None, jobs=0,
directory=None, verbose=False, keep_going=False, mach_context=None):
"""Invoke the build backend.
@ -985,6 +989,7 @@ class BuildDriver(MozbuildObject):
``what`` defines the thing to build. If not defined, the default
target is used.
"""
self.mach_context = mach_context
warnings_path = self._get_state_filename('warnings.json')
monitor = self._spawn(BuildMonitor)
monitor.init(warnings_path)
@ -1054,11 +1059,15 @@ class BuildDriver(MozbuildObject):
monitor.start_resource_recording()
self.mach_context.command_attrs['clobber'] = False
config = None
try:
config = self.config_environment
except Exception:
pass
# If we don't already have a config environment this is either
# a fresh objdir or $OBJDIR/config.status has been removed for
# some reason, which indicates a clobber of sorts.
self.mach_context.command_attrs['clobber'] = True
# Record whether a clobber was requested so we can print
# a special message later if the build fails.
@ -1227,6 +1236,11 @@ class BuildDriver(MozbuildObject):
monitor.finish(record_usage=status == 0)
if status == 0:
usage = monitor.get_resource_usage()
if usage:
self.mach_context.command_attrs['usage'] = usage
# Print the collected compiler warnings. This is redundant with
# inline output from the compiler itself. However, unlike inline
# output, this list is sorted and grouped by file, making it
@ -1535,6 +1549,8 @@ class BuildDriver(MozbuildObject):
{'msg': line.rstrip()}, '{msg}')
clobber_required, clobber_performed, clobber_message = res
if self.mach_context is not None and clobber_performed:
self.mach_context.command_attrs['clobber'] = True
if not clobber_required or clobber_performed:
if clobber_performed and env.get('TINDERBOX_OUTPUT'):
self.log(logging.WARNING, 'clobber',

View File

@ -228,6 +228,19 @@ def get_build_opts(substs):
return {}
def get_build_attrs(attrs):
'''
Extracts clobber and cpu usage info from command attributes.
'''
res = {}
clobber = attrs.get('clobber')
if clobber:
res['clobber'] = clobber
usage = attrs.get('usage')
if usage:
res['cpu_percent'] = int(round(usage['cpu_percent']))
return res
def filter_args(command, argv, paths):
'''
Given the full list of command-line arguments, remove anything up to and including `command`,
@ -253,7 +266,7 @@ def filter_args(command, argv, paths):
def gather_telemetry(command='', success=False, start_time=None, end_time=None,
mach_context=None, substs={}, paths={}):
mach_context=None, substs={}, paths={}, command_attrs=None):
'''
Gather telemetry about the build and the user's system and pass it to the telemetry
handler to be stored for later submission.
@ -274,6 +287,7 @@ def gather_telemetry(command='', success=False, start_time=None, end_time=None,
# TODO: use a monotonic clock: https://bugzilla.mozilla.org/show_bug.cgi?id=1481624
'duration_ms': int((end_time - start_time) * 1000),
'build_opts': get_build_opts(substs),
'build_attrs': get_build_attrs(command_attrs),
'system': get_system_info(),
# TODO: exception: https://bugzilla.mozilla.org/show_bug.cgi?id=1481617
# TODO: file_types_changed: https://bugzilla.mozilla.org/show_bug.cgi?id=1481774