mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-10-19 08:15:31 +00:00
Bug 1486789 - support subtest units independent of benchmark units, set values for speedometer. r=ahal
allow support in .ini files for subtest units and lower_is_better values; implement this for speedometer. Differential Revision: https://phabricator.services.mozilla.com/D4465 --HG-- extra : moz-landing-system : lando
This commit is contained in:
Joel Maher
parent
a6b4ae3b60
commit
49ed6e30bf
@ -89,6 +89,13 @@ def write_test_settings_json(test_details, oskey):
|
||||
test_settings['raptor-options']['lower_is_better'] = False
|
||||
else:
|
||||
test_settings['raptor-options']['lower_is_better'] = True
|
||||
|
||||
# support optional subtest unit/lower_is_better fields, default to main test values if not set
|
||||
val = test_details.get('subtest_unit', test_settings['raptor-options']['unit'])
|
||||
test_settings['raptor-options']['subtest_unit'] = val
|
||||
val = test_details.get('subtest_lower', test_settings['raptor-options']['lower_is_better'])
|
||||
test_settings['raptor-options']['subtest_lower_is_better'] = val
|
||||
|
||||
if test_details.get("alert_threshold", None) is not None:
|
||||
test_settings['raptor-options']['alert_threshold'] = float(test_details['alert_threshold'])
|
||||
|
||||
|
||||
@ -51,6 +51,7 @@ class Output(object):
|
||||
'extraOptions': test.extra_options,
|
||||
'subtests': subtests,
|
||||
'lowerIsBetter': test.lower_is_better,
|
||||
'unit': test.unit,
|
||||
'alertThreshold': float(test.alert_threshold)
|
||||
}
|
||||
|
||||
@ -75,10 +76,10 @@ class Output(object):
|
||||
new_subtest = {}
|
||||
new_subtest['name'] = test.name + "-" + measurement_name
|
||||
new_subtest['replicates'] = replicates
|
||||
new_subtest['lowerIsBetter'] = test.lower_is_better
|
||||
new_subtest['lowerIsBetter'] = test.subtest_lower_is_better
|
||||
new_subtest['alertThreshold'] = float(test.alert_threshold)
|
||||
new_subtest['value'] = 0
|
||||
new_subtest['unit'] = test.unit
|
||||
new_subtest['unit'] = test.subtest_unit
|
||||
|
||||
filtered_values = filter.ignore_first(new_subtest['replicates'], 1)
|
||||
new_subtest['value'] = filter.median(filtered_values)
|
||||
@ -146,9 +147,9 @@ class Output(object):
|
||||
# for each pagecycle, build a list of subtests and append all related replicates
|
||||
if sub not in _subtests.keys():
|
||||
# subtest not added yet, first pagecycle, so add new one
|
||||
_subtests[sub] = {'unit': test.unit,
|
||||
_subtests[sub] = {'unit': test.subtest_unit,
|
||||
'alertThreshold': float(test.alert_threshold),
|
||||
'lowerIsBetter': test.lower_is_better,
|
||||
'lowerIsBetter': test.subtest_lower_is_better,
|
||||
'name': sub,
|
||||
'replicates': []}
|
||||
_subtests[sub]['replicates'].extend([round(x, 3) for x in replicates])
|
||||
@ -183,9 +184,9 @@ class Output(object):
|
||||
sub = item['name']
|
||||
if sub not in _subtests.keys():
|
||||
# subtest not added yet, first pagecycle, so add new one
|
||||
_subtests[sub] = {'unit': test.unit,
|
||||
_subtests[sub] = {'unit': test.subtest_unit,
|
||||
'alertThreshold': float(test.alert_threshold),
|
||||
'lowerIsBetter': test.lower_is_better,
|
||||
'lowerIsBetter': test.subtest_lower_is_better,
|
||||
'name': sub,
|
||||
'replicates': []}
|
||||
_subtests[sub]['replicates'].append(item['time'])
|
||||
@ -230,9 +231,9 @@ class Output(object):
|
||||
replicates = [item['duration']]
|
||||
if sub not in _subtests.keys():
|
||||
# subtest not added yet, first pagecycle, so add new one
|
||||
_subtests[sub] = {'unit': test.unit,
|
||||
_subtests[sub] = {'unit': test.subtest_unit,
|
||||
'alertThreshold': float(test.alert_threshold),
|
||||
'lowerIsBetter': test.lower_is_better,
|
||||
'lowerIsBetter': test.subtest_lower_is_better,
|
||||
'name': sub,
|
||||
'replicates': []}
|
||||
_subtests[sub]['replicates'].extend([round(x, 3) for x in replicates])
|
||||
@ -285,9 +286,9 @@ class Output(object):
|
||||
|
||||
if sub not in _subtests.keys():
|
||||
# subtest not added yet, first pagecycle, so add new one
|
||||
_subtests[sub] = {'unit': test.unit,
|
||||
_subtests[sub] = {'unit': test.subtest_unit,
|
||||
'alertThreshold': float(test.alert_threshold),
|
||||
'lowerIsBetter': test.lower_is_better,
|
||||
'lowerIsBetter': test.subtest_lower_is_better,
|
||||
'name': sub,
|
||||
'replicates': []}
|
||||
_subtests[sub]['replicates'].extend([replicate])
|
||||
@ -311,17 +312,17 @@ class Output(object):
|
||||
# for each pagecycle, build a list of subtests and append all related replicates
|
||||
if sub not in _subtests.keys():
|
||||
# subtest not added yet, first pagecycle, so add new one
|
||||
_subtests[sub] = {'unit': test.unit,
|
||||
_subtests[sub] = {'unit': test.subtest_unit,
|
||||
'alertThreshold': float(test.alert_threshold),
|
||||
'lowerIsBetter': test.lower_is_better,
|
||||
'lowerIsBetter': test.subtest_lower_is_better,
|
||||
'name': sub,
|
||||
'replicates': []}
|
||||
_subtests[sub]['replicates'].extend([round(x, 3) for x in replicates])
|
||||
|
||||
total_subtest = {
|
||||
'unit': test.unit,
|
||||
'unit': test.subtest_unit,
|
||||
'alertThreshold': float(test.alert_threshold),
|
||||
'lowerIsBetter': test.lower_is_better,
|
||||
'lowerIsBetter': test.subtest_lower_is_better,
|
||||
'replicates': [],
|
||||
'name': 'benchmark_score',
|
||||
'value': 0
|
||||
@ -365,9 +366,9 @@ class Output(object):
|
||||
sub = item['benchmark']
|
||||
if sub not in _subtests.keys():
|
||||
# subtest not added yet, first pagecycle, so add new one
|
||||
_subtests[sub] = {'unit': test.unit,
|
||||
_subtests[sub] = {'unit': test.subtest_unit,
|
||||
'alertThreshold': float(test.alert_threshold),
|
||||
'lowerIsBetter': test.lower_is_better,
|
||||
'lowerIsBetter': test.subtest_lower_is_better,
|
||||
'name': sub,
|
||||
'replicates': []}
|
||||
_subtests[sub]['replicates'].append(item['result'])
|
||||
@ -404,9 +405,9 @@ class Output(object):
|
||||
# build a list of subtests and append all related replicates
|
||||
if _sub not in _subtests.keys():
|
||||
# subtest not added yet, first pagecycle, so add new one
|
||||
_subtests[_sub] = {'unit': test.unit,
|
||||
_subtests[_sub] = {'unit': test.subtest_unit,
|
||||
'alertThreshold': float(test.alert_threshold),
|
||||
'lowerIsBetter': test.lower_is_better,
|
||||
'lowerIsBetter': test.subtest_lower_is_better,
|
||||
'name': _sub,
|
||||
'replicates': []}
|
||||
_subtests[_sub]['replicates'].extend([_value])
|
||||
|
||||
@ -9,8 +9,10 @@ type = benchmark
|
||||
test_url = http://localhost:<port>/Speedometer/index.html?raptor
|
||||
page_cycles = 5
|
||||
page_timeout = 180000
|
||||
unit = score
|
||||
lower_is_better = false
|
||||
unit = score
|
||||
subtest_unit = ms
|
||||
lower_is_better = false
|
||||
subtest_lower_is_better = true
|
||||
alert_threshold = 2.0
|
||||
|
||||
[raptor-speedometer-firefox]
|
||||
|
||||
@ -78,7 +78,9 @@ function getTestSettings() {
|
||||
results.type = testType;
|
||||
results.name = testName;
|
||||
results.unit = settings.unit;
|
||||
results.lower_is_better = settings.lower_is_better;
|
||||
results.subtest_unit = settings.subtest_unit;
|
||||
results.lower_is_better = settings.lower_is_better == "true";
|
||||
results.subtest_lower_is_better = settings.subtest_lower_is_better == "true";
|
||||
results.alert_threshold = settings.alert_threshold;
|
||||
|
||||
if (settings.page_timeout !== undefined) {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user