mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-10-24 02:35:41 +00:00
Bug 1567311 - Consolidate supporting raptor measurements into one perfherder output and artifact per type. r=perftest-reviewers,rwood
This patch fixes four things at once: (1) Consolidate supporting raptor measurements into one PERFHERDER_DATA output per type. For example, with this change, all power data will be grouped into one PERFHERDER_DATA output. (2) Output perfherder-data-<DATA_TYPE>.json for each supporting measurement instead of overwriting perfherder-data.json which contains the regular raptor test results. (3) Take an average of the supporting measurements when particular unit's are specified. In this case, the '%' unit makes us take the average instead of the sum of the measurements. (4) Remove the redundant test name entry that prefixes all power subtest entries. Differential Revision: https://phabricator.services.mozilla.com/D40667 --HG-- extra : moz-landing-system : lando
This commit is contained in:
parent
fb977a3526
commit
36210f5278
@ -6,6 +6,7 @@ from __future__ import absolute_import, print_function, unicode_literals
|
|||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import copy
|
import copy
|
||||||
|
import glob
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
@ -680,16 +681,24 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin):
|
|||||||
self.info(str(dest))
|
self.info(str(dest))
|
||||||
self._artifact_perf_data(src, dest)
|
self._artifact_perf_data(src, dest)
|
||||||
|
|
||||||
if self.power_test:
|
# make individual perfherder data json's for each supporting data type
|
||||||
src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'raptor-power.json')
|
for file in glob.glob(os.path.join(self.query_abs_dirs()['abs_work_dir'], '*')):
|
||||||
self._artifact_perf_data(src, dest)
|
path, filename = os.path.split(file)
|
||||||
|
|
||||||
if self.memory_test:
|
if not filename.startswith('raptor-'):
|
||||||
src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'raptor-memory.json')
|
continue
|
||||||
self._artifact_perf_data(src, dest)
|
|
||||||
|
|
||||||
if self.cpu_test:
|
# filename is expected to contain a unique data name
|
||||||
src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'raptor-cpu.json')
|
# i.e. raptor-os-baseline-power.json would result in
|
||||||
|
# the data name os-baseline-power
|
||||||
|
data_name = '-'.join(filename.split('-')[1:])
|
||||||
|
data_name = '.'.join(data_name.split('.')[:-1])
|
||||||
|
|
||||||
|
src = file
|
||||||
|
dest = os.path.join(
|
||||||
|
env['MOZ_UPLOAD_DIR'],
|
||||||
|
'perfherder-data-%s.json' % data_name
|
||||||
|
)
|
||||||
self._artifact_perf_data(src, dest)
|
self._artifact_perf_data(src, dest)
|
||||||
|
|
||||||
src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'screenshots.html')
|
src = os.path.join(self.query_abs_dirs()['abs_work_dir'], 'screenshots.html')
|
||||||
|
@ -322,20 +322,21 @@ class Output(object):
|
|||||||
return
|
return
|
||||||
|
|
||||||
self.summarized_supporting_data = []
|
self.summarized_supporting_data = []
|
||||||
|
support_data_by_type = {}
|
||||||
|
|
||||||
for data_set in self.supporting_data:
|
for data_set in self.supporting_data:
|
||||||
|
|
||||||
suites = []
|
|
||||||
test_results = {
|
|
||||||
'framework': {
|
|
||||||
'name': 'raptor',
|
|
||||||
},
|
|
||||||
'suites': suites,
|
|
||||||
}
|
|
||||||
|
|
||||||
data_type = data_set['type']
|
data_type = data_set['type']
|
||||||
LOG.info("summarizing %s data" % data_type)
|
LOG.info("summarizing %s data" % data_type)
|
||||||
|
|
||||||
|
if data_type not in support_data_by_type:
|
||||||
|
support_data_by_type[data_type] = {
|
||||||
|
'framework': {
|
||||||
|
'name': 'raptor',
|
||||||
|
},
|
||||||
|
'suites': [],
|
||||||
|
}
|
||||||
|
|
||||||
# suite name will be name of the actual raptor test that ran, plus the type of
|
# suite name will be name of the actual raptor test that ran, plus the type of
|
||||||
# supporting data i.e. 'raptor-speedometer-geckoview-power'
|
# supporting data i.e. 'raptor-speedometer-geckoview-power'
|
||||||
vals = []
|
vals = []
|
||||||
@ -349,14 +350,16 @@ class Output(object):
|
|||||||
'alertThreshold': 2.0
|
'alertThreshold': 2.0
|
||||||
}
|
}
|
||||||
|
|
||||||
suites.append(suite)
|
support_data_by_type[data_type]['suites'].append(suite)
|
||||||
|
|
||||||
# each supporting data measurement becomes a subtest, with the measurement type
|
# each supporting data measurement becomes a subtest, with the measurement type
|
||||||
# used for the subtest name. i.e. 'raptor-speedometer-geckoview-power-cpu'
|
# used for the subtest name. i.e. 'power-cpu'
|
||||||
# the overall 'suite' value for supporting data will be the sum of all measurements
|
# the overall 'suite' value for supporting data is dependent on
|
||||||
|
# the unit of the values, by default the sum of all measurements
|
||||||
|
# is taken.
|
||||||
for measurement_name, value in data_set['values'].iteritems():
|
for measurement_name, value in data_set['values'].iteritems():
|
||||||
new_subtest = {}
|
new_subtest = {}
|
||||||
new_subtest['name'] = data_set['test'] + "-" + data_type + "-" + measurement_name
|
new_subtest['name'] = data_type + "-" + measurement_name
|
||||||
new_subtest['value'] = value
|
new_subtest['value'] = value
|
||||||
new_subtest['lowerIsBetter'] = True
|
new_subtest['lowerIsBetter'] = True
|
||||||
new_subtest['alertThreshold'] = 2.0
|
new_subtest['alertThreshold'] = 2.0
|
||||||
@ -364,10 +367,17 @@ class Output(object):
|
|||||||
subtests.append(new_subtest)
|
subtests.append(new_subtest)
|
||||||
vals.append([new_subtest['value'], new_subtest['name']])
|
vals.append([new_subtest['value'], new_subtest['name']])
|
||||||
|
|
||||||
if len(subtests) > 1:
|
if len(subtests) >= 1:
|
||||||
suite['value'] = self.construct_summary(vals, testname="supporting_data")
|
suite['value'] = self.construct_summary(
|
||||||
|
vals,
|
||||||
|
testname="supporting_data",
|
||||||
|
unit=data_set['unit']
|
||||||
|
)
|
||||||
|
|
||||||
self.summarized_supporting_data.append(test_results)
|
# split the supporting data by type, there will be one
|
||||||
|
# perfherder output per type
|
||||||
|
for data_type in support_data_by_type:
|
||||||
|
self.summarized_supporting_data.append(support_data_by_type[data_type])
|
||||||
|
|
||||||
return
|
return
|
||||||
|
|
||||||
@ -1154,7 +1164,12 @@ class Output(object):
|
|||||||
results = [i for i, j in val_list]
|
results = [i for i, j in val_list]
|
||||||
return sum(results)
|
return sum(results)
|
||||||
|
|
||||||
def construct_summary(self, vals, testname):
|
@classmethod
|
||||||
|
def supporting_data_average(cls, val_list):
|
||||||
|
results = [i for i, j in val_list]
|
||||||
|
return sum(results)/len(results)
|
||||||
|
|
||||||
|
def construct_summary(self, vals, testname, unit=None):
|
||||||
if testname.startswith('raptor-v8_7'):
|
if testname.startswith('raptor-v8_7'):
|
||||||
return self.v8_Metric(vals)
|
return self.v8_Metric(vals)
|
||||||
elif testname.startswith('raptor-kraken'):
|
elif testname.startswith('raptor-kraken'):
|
||||||
@ -1180,7 +1195,10 @@ class Output(object):
|
|||||||
elif testname.startswith('raptor-youtube-playback'):
|
elif testname.startswith('raptor-youtube-playback'):
|
||||||
return self.youtube_playback_performance_score(vals)
|
return self.youtube_playback_performance_score(vals)
|
||||||
elif testname.startswith('supporting_data'):
|
elif testname.startswith('supporting_data'):
|
||||||
return self.supporting_data_total(vals)
|
if unit and unit in ('%',):
|
||||||
|
return self.supporting_data_average(vals)
|
||||||
|
else:
|
||||||
|
return self.supporting_data_total(vals)
|
||||||
elif len(vals) > 1:
|
elif len(vals) > 1:
|
||||||
return round(filters.geometric_mean([i for i, j in vals]), 2)
|
return round(filters.geometric_mean([i for i, j in vals]), 2)
|
||||||
else:
|
else:
|
||||||
|
@ -96,10 +96,6 @@ class RaptorResultsHandler():
|
|||||||
if data_type == 'scenario':
|
if data_type == 'scenario':
|
||||||
return None
|
return None
|
||||||
|
|
||||||
if self.config.get('power_test', None):
|
|
||||||
# Add one for OS baseline, and one for %change
|
|
||||||
# measurement
|
|
||||||
expected_perfherder += 2
|
|
||||||
if self.config.get('memory_test', None):
|
if self.config.get('memory_test', None):
|
||||||
expected_perfherder += 1
|
expected_perfherder += 1
|
||||||
if self.config.get('cpu_test', None):
|
if self.config.get('cpu_test', None):
|
||||||
|
Loading…
Reference in New Issue
Block a user