Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-09 03:15:11 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1831856 - Replace median with geomean for performance tests r=perftest-reviewers,sparky

Browse Source 
Differential Revision: https://phabricator.services.mozilla.com/D177374
This commit is contained in:
Alex Ionescu 2023-06-06 07:42:37 +00:00
parent 8d6eb53126
commit 57a6e8d7bb
5 changed files with 13 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
taskcluster/ci/test/browsertime-desktop.yml
View File

@ -96,12 +96,10 @@ browsertime-tp6:
- --browsertime-no-ffwindowrecorder
- --conditioned-profile=settled
- --extra-profiler-run
- --extra-summary-methods=geomean
default:
- --chimera
- --conditioned-profile=settled
- --extra-profiler-run
- --extra-summary-methods=geomean
run-on-projects:
by-subtest:
netflix: # Bug 1756212 - Disable temporarily due to perma on mac
@ -204,12 +202,10 @@ browsertime-tp6-essential:
- --conditioned-profile=settled
- --collect-perfstats
- --extra-profiler-run
- --extra-summary-methods=geomean
default:
- --chimera
- --conditioned-profile=settled
- --extra-profiler-run
- --extra-summary-methods=geomean
raptor:
apps: [firefox, chrome, chromium, custom-car]
test: tp6

1
taskcluster/ci/test/browsertime-mobile.yml
View File

@ -76,7 +76,6 @@ browsertime-tp6m:
mozharness:
extra-options:
- --chimera
- --extra-summary-methods=geomean
tier:
by-app:
geckoview: 1

2
testing/mozharness/mozharness/mozilla/testing/raptor.py
View File

@ -610,7 +610,7 @@ class Raptor(
"help": (
"Alternative methods for summarizing technical and visual"
"pageload metrics."
"Options: geomean, mean."
"Options: median."
),
},
],

2
testing/raptor/raptor/cmdline.py
View File

@ -511,7 +511,7 @@ def create_parser(mach_interface=False):
default=[],
metavar="OPTION",
help="Alternative methods for summarizing technical and visual pageload metrics. "
"Options: geomean, mean.",
"Options: median.",
)
add_arg(
"--benchmark-repository",

35
testing/raptor/raptor/output.py
View File

@ -1695,36 +1695,23 @@ class BrowsertimeOutput(PerftestOutput):
return data
def _process_alt_method(subtest, alternative_method):
# Don't filter with less than 10 data points
def _process_geomean(subtest):
data = subtest["replicates"]
if len(subtest["replicates"]) > 10:
data = _filter_data(data, alternative_method, subtest["name"])
if alternative_method == "geomean":
subtest["value"] = round(filters.geometric_mean(data), 1)
elif alternative_method == "mean":
subtest["value"] = round(filters.mean(data), 1)
subtest["value"] = round(filters.geometric_mean(data), 1)
def _process_alt_method(subtest, alternative_method):
data = subtest["replicates"]
if alternative_method == "median":
subtest["value"] = filters.median(data)
# converting suites and subtests into lists, and sorting them
def _process(subtest, alternative_method=""):
def _process(subtest, method="geomean"):
if test["type"] == "power":
subtest["value"] = filters.mean(subtest["replicates"])
elif (
subtest["name"] in VISUAL_METRICS
or subtest["name"].startswith("perfstat")
or subtest["name"] == "cpuTime"
):
if alternative_method in ("geomean", "mean"):
_process_alt_method(subtest, alternative_method)
else:
subtest["value"] = filters.median(subtest["replicates"])
elif method == "geomean":
_process_geomean(subtest)
else:
if alternative_method in ("geomean", "mean"):
_process_alt_method(subtest, alternative_method)
else:
subtest["value"] = filters.median(
filters.ignore_first(subtest["replicates"], 1)
)
_process_alt_method(subtest, method)
return subtest
def _process_suite(suite):