Bug 1526072 - Record cpu utilization and clobber/full builds in build telemetry. r=nalexander

Differential Revision: https://phabricator.services.mozilla.com/D22630

--HG--
extra : moz-landing-system : lando

build/mach_bootstrap.py

@@ -250,6 +250,8 @@ def bootstrap(topsrcdir, mozilla_dir=None):
         except Exception:
             substs = {}
 
+        command_attrs = getattr(context, 'command_attrs', {})
+
         # We gather telemetry for every operation.
         paths = {
             instance.topsrcdir: '$topsrcdir/',
@@ -263,7 +265,7 @@ def bootstrap(topsrcdir, mozilla_dir=None):
         data = gather_telemetry(command=handler.name, success=(result == 0),
                                 start_time=start_time, end_time=end_time,
                                 mach_context=context, substs=substs,
-                                paths=paths)
+                                command_attrs=command_attrs, paths=paths)
         if data:
             telemetry_dir = os.path.join(get_state_dir(), 'telemetry')
             try:

python/mach/mach/base.py

@@ -12,6 +12,7 @@ class CommandContext(object):
         self.settings = settings
         self.log_manager = log_manager
         self.commands = commands
+        self.command_attrs = {}
 
         for k, v in kwargs.items():
             setattr(self, k, v)

python/mozbuild/mozbuild/controller/building.py

@@ -978,6 +978,10 @@ class CCacheStats(object):
 class BuildDriver(MozbuildObject):
     """Provides a high-level API for build actions."""
 
+    def __init__(self, *args, **kwargs):
+        MozbuildObject.__init__(self, *args, **kwargs)
+        self.mach_context = None
+
     def build(self, what=None, disable_extra_make_dependencies=None, jobs=0,
               directory=None, verbose=False, keep_going=False, mach_context=None):
         """Invoke the build backend.
@@ -985,6 +989,7 @@ class BuildDriver(MozbuildObject):
         ``what`` defines the thing to build. If not defined, the default
         target is used.
         """
+        self.mach_context = mach_context
         warnings_path = self._get_state_filename('warnings.json')
         monitor = self._spawn(BuildMonitor)
         monitor.init(warnings_path)
@@ -1054,11 +1059,15 @@ class BuildDriver(MozbuildObject):
 
             monitor.start_resource_recording()
 
+            self.mach_context.command_attrs['clobber'] = False
             config = None
             try:
                 config = self.config_environment
             except Exception:
-                pass
+                # If we don't already have a config environment this is either
+                # a fresh objdir or $OBJDIR/config.status has been removed for
+                # some reason, which indicates a clobber of sorts.
+                self.mach_context.command_attrs['clobber'] = True
 
             # Record whether a clobber was requested so we can print
             # a special message later if the build fails.
@@ -1227,6 +1236,11 @@ class BuildDriver(MozbuildObject):
 
             monitor.finish(record_usage=status == 0)
 
+        if status == 0:
+            usage = monitor.get_resource_usage()
+            if usage:
+                self.mach_context.command_attrs['usage'] = usage
+
         # Print the collected compiler warnings. This is redundant with
         # inline output from the compiler itself. However, unlike inline
         # output, this list is sorted and grouped by file, making it
@@ -1535,6 +1549,8 @@ class BuildDriver(MozbuildObject):
                      {'msg': line.rstrip()}, '{msg}')
 
         clobber_required, clobber_performed, clobber_message = res
+        if self.mach_context is not None and clobber_performed:
+            self.mach_context.command_attrs['clobber'] = True
         if not clobber_required or clobber_performed:
             if clobber_performed and env.get('TINDERBOX_OUTPUT'):
                 self.log(logging.WARNING, 'clobber',

python/mozbuild/mozbuild/telemetry.py

@@ -228,6 +228,19 @@ def get_build_opts(substs):
         return {}
 
 
+def get_build_attrs(attrs):
+    '''
+    Extracts clobber and cpu usage info from command attributes.
+    '''
+    res = {}
+    clobber = attrs.get('clobber')
+    if clobber:
+        res['clobber'] = clobber
+    usage = attrs.get('usage')
+    if usage:
+        res['cpu_percent'] = int(round(usage['cpu_percent']))
+    return res
+
 def filter_args(command, argv, paths):
     '''
     Given the full list of command-line arguments, remove anything up to and including `command`,
@@ -253,7 +266,7 @@ def filter_args(command, argv, paths):
 
 
 def gather_telemetry(command='', success=False, start_time=None, end_time=None,
-                     mach_context=None, substs={}, paths={}):
+                     mach_context=None, substs={}, paths={}, command_attrs=None):
     '''
     Gather telemetry about the build and the user's system and pass it to the telemetry
     handler to be stored for later submission.
@@ -274,6 +287,7 @@ def gather_telemetry(command='', success=False, start_time=None, end_time=None,
         # TODO: use a monotonic clock: https://bugzilla.mozilla.org/show_bug.cgi?id=1481624
         'duration_ms': int((end_time - start_time) * 1000),
         'build_opts': get_build_opts(substs),
+        'build_attrs': get_build_attrs(command_attrs),
         'system': get_system_info(),
         # TODO: exception: https://bugzilla.mozilla.org/show_bug.cgi?id=1481617
         # TODO: file_types_changed: https://bugzilla.mozilla.org/show_bug.cgi?id=1481774