Bug 1567311 - Consolidate supporting raptor measurements into one perfherder output and artifact per type. r=perftest-reviewers,rwood

This patch fixes four things at once:

(1) Consolidate supporting raptor measurements into one PERFHERDER_DATA output per type. For example, with this change, all power data will be grouped into one PERFHERDER_DATA output.

(2) Output perfherder-data-<DATA_TYPE>.json for each supporting measurement instead of overwriting perfherder-data.json which contains the regular raptor test results.

(3) Take an average of the supporting measurements when particular unit's are specified. In this case, the '%' unit makes us take the average instead of the sum of the measurements.

(4) Remove the redundant test name entry that prefixes all power subtest entries.

Differential Revision: https://phabricator.services.mozilla.com/D40667

--HG--
extra : moz-landing-system : lando

testing/mozharness/mozharness/mozilla/testing/raptor.py

@@ -6,6 +6,7 @@ from __future__ import absolute_import, print_function, unicode_literals
 
 import argparse
 import copy
+import glob
 import os
 import re
 import sys
@@ -680,16 +681,24 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin):
             self.info(str(dest))
             self._artifact_perf_data(src, dest)
 
-            if self.power_test:
+            # make individual perfherder data json's for each supporting data type
-                src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'raptor-power.json')
+            for file in glob.glob(os.path.join(self.query_abs_dirs()['abs_work_dir'], '*')):
-                self._artifact_perf_data(src, dest)
+                path, filename = os.path.split(file)
 
-            if self.memory_test:
+                if not filename.startswith('raptor-'):
-                src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'raptor-memory.json')
+                    continue
-                self._artifact_perf_data(src, dest)
 
-            if self.cpu_test:
+                # filename is expected to contain a unique data name
-                src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'raptor-cpu.json')
+                # i.e. raptor-os-baseline-power.json would result in
+                # the data name os-baseline-power
+                data_name = '-'.join(filename.split('-')[1:])
+                data_name = '.'.join(data_name.split('.')[:-1])
+
+                src = file
+                dest = os.path.join(
+                    env['MOZ_UPLOAD_DIR'],
+                    'perfherder-data-%s.json' % data_name
+                )
                 self._artifact_perf_data(src, dest)
 
             src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'screenshots.html')

testing/raptor/raptor/output.py

@@ -322,20 +322,21 @@ class Output(object):
             return
 
         self.summarized_supporting_data = []
+        support_data_by_type = {}
 
         for data_set in self.supporting_data:
 
-            suites = []
-            test_results = {
-                'framework': {
-                    'name': 'raptor',
-                },
-                'suites': suites,
-            }
-
             data_type = data_set['type']
             LOG.info("summarizing %s data" % data_type)
 
+            if data_type not in support_data_by_type:
+                support_data_by_type[data_type] = {
+                    'framework': {
+                        'name': 'raptor',
+                    },
+                    'suites': [],
+                }
+
             # suite name will be name of the actual raptor test that ran, plus the type of
             # supporting data i.e. 'raptor-speedometer-geckoview-power'
             vals = []
@@ -349,14 +350,16 @@ class Output(object):
                 'alertThreshold': 2.0
             }
 
-            suites.append(suite)
+            support_data_by_type[data_type]['suites'].append(suite)
 
             # each supporting data measurement becomes a subtest, with the measurement type
-            # used for the subtest name. i.e. 'raptor-speedometer-geckoview-power-cpu'
+            # used for the subtest name. i.e. 'power-cpu'
-            # the overall 'suite' value for supporting data will be the sum of all measurements
+            # the overall 'suite' value for supporting data is dependent on
+            # the unit of the values, by default the sum of all measurements
+            # is taken.
             for measurement_name, value in data_set['values'].iteritems():
                 new_subtest = {}
-                new_subtest['name'] = data_set['test'] + "-" + data_type + "-" + measurement_name
+                new_subtest['name'] = data_type + "-" + measurement_name
                 new_subtest['value'] = value
                 new_subtest['lowerIsBetter'] = True
                 new_subtest['alertThreshold'] = 2.0
@@ -364,10 +367,17 @@ class Output(object):
                 subtests.append(new_subtest)
                 vals.append([new_subtest['value'], new_subtest['name']])
 
-            if len(subtests) > 1:
+            if len(subtests) >= 1:
-                suite['value'] = self.construct_summary(vals, testname="supporting_data")
+                suite['value'] = self.construct_summary(
+                    vals,
+                    testname="supporting_data",
+                    unit=data_set['unit']
+                )
 
-            self.summarized_supporting_data.append(test_results)
+        # split the supporting data by type, there will be one
+        # perfherder output per type
+        for data_type in support_data_by_type:
+            self.summarized_supporting_data.append(support_data_by_type[data_type])
 
         return
 
@@ -1154,7 +1164,12 @@ class Output(object):
         results = [i for i, j in val_list]
         return sum(results)
 
-    def construct_summary(self, vals, testname):
+    @classmethod
+    def supporting_data_average(cls, val_list):
+        results = [i for i, j in val_list]
+        return sum(results)/len(results)
+
+    def construct_summary(self, vals, testname, unit=None):
         if testname.startswith('raptor-v8_7'):
             return self.v8_Metric(vals)
         elif testname.startswith('raptor-kraken'):
@@ -1180,7 +1195,10 @@ class Output(object):
         elif testname.startswith('raptor-youtube-playback'):
             return self.youtube_playback_performance_score(vals)
         elif testname.startswith('supporting_data'):
-            return self.supporting_data_total(vals)
+            if unit and unit in ('%',):
+                return self.supporting_data_average(vals)
+            else:
+                return self.supporting_data_total(vals)
         elif len(vals) > 1:
             return round(filters.geometric_mean([i for i, j in vals]), 2)
         else:

testing/raptor/raptor/results.py

@@ -96,10 +96,6 @@ class RaptorResultsHandler():
                 if data_type == 'scenario':
                     return None
 
-        if self.config.get('power_test', None):
-            # Add one for OS baseline, and one for %change
-            # measurement
-            expected_perfherder += 2
         if self.config.get('memory_test', None):
             expected_perfherder += 1
         if self.config.get('cpu_test', None):