Bug 1692461 - Migrate wasm-godot to browsertime desktop r=perftest-reviewers,Bebe

Latest build: https://treeherder.mozilla.org/jobs?repo=try&tier=1%2C2%2C3&revision=96e7ed7dcccef459465e6eab85884a8e0990ff5a Please leave the landing to the author of the patch! Differential Revision: https://phabricator.services.mozilla.com/D106934
2024-11-24 13:21:05 +00:00 · 2021-03-25 12:29:07 +00:00 · 2021-03-25 12:29:07 +00:00 · 06a35153dd
commit 06a35153dd
parent 6a17b62da7
8 changed files with 126 additions and 41 deletions
--- a/taskcluster/ci/test/browsertime-desktop.yml
+++ b/taskcluster/ci/test/browsertime-desktop.yml
@ -143,6 +143,9 @@ browsertime-benchmark:
        - [motionmark-htmlsuite, mm-h]
        - [stylebench, sb]
        - [webaudio, wa]
+        - [wasm-godot, godot]
+        - [wasm-godot-baseline, godot-b]
+        - [wasm-godot-optimizing, godot-i]
    variants: ["fission"]
    apps: ["firefox"]
    treeherder-symbol: Btime()
--- a/taskcluster/ci/test/raptor.yml
+++ b/taskcluster/ci/test/raptor.yml
@ -306,6 +306,7 @@ raptor-wasm-misc-firefox:
    variants: ["fission"]
    try-name: raptor-wasm-misc-firefox
    treeherder-symbol: Rap(wm)
+    run-on-projects: []
    mozharness:
        extra-options:
            - --test=raptor-wasm-misc
@ -317,6 +318,7 @@ raptor-wasm-misc-firefox-profiling:
    description: "Raptor WASM Misc on Firefox with Gecko Profiling"
    try-name: raptor-wasm-misc-firefox-profiling
    treeherder-symbol: Rap-Prof(wm)
+    run-on-projects: []
    max-run-time: 900
    mozharness:
        extra-options:
@ -331,6 +333,7 @@ raptor-wasm-misc-baseline-firefox:
    variants: ["fission"]
    try-name: raptor-wasm-misc-baseline-firefox
    treeherder-symbol: Rap(wm-b)
+    run-on-projects: []
    mozharness:
        extra-options:
            - --test=raptor-wasm-misc-baseline
@ -342,6 +345,7 @@ raptor-wasm-misc-baseline-firefox-profiling:
    description: "Raptor WASM Misc on Firefox with baseline JIT and Gecko Profiling"
    try-name: raptor-wasm-misc-baseline-firefox-profiling
    treeherder-symbol: Rap-Prof(wm-b)
+    run-on-projects: []
    max-run-time: 900
    mozharness:
        extra-options:
@ -356,6 +360,7 @@ raptor-wasm-misc-optimizing-firefox:
    variants: ["fission"]
    try-name: raptor-wasm-misc-optimizing-firefox
    treeherder-symbol: Rap(wm-i)
+    run-on-projects: []
    mozharness:
        extra-options:
            - --test=raptor-wasm-misc-optimizing
@ -367,6 +372,7 @@ raptor-wasm-misc-optimizing-firefox-profiling:
    description: "Raptor WASM Misc on Firefox with optimizing JIT and Gecko Profiling"
    try-name: raptor-wasm-misc-optimizing-firefox-profiling
    treeherder-symbol: Rap-Prof(wm-i)
+    run-on-projects: []
    max-run-time: 900
    mozharness:
        extra-options:
@ -411,6 +417,7 @@ raptor-wasm-godot-firefox:
    variants: ["fission"]
    try-name: raptor-wasm-godot-firefox
    treeherder-symbol: Rap(godot)
+    run-on-projects: []
    max-run-time:
        by-test-platform:
            .*-qr/.*: 2100
@ -424,6 +431,7 @@ raptor-wasm-godot-firefox-profiling:
    description: "Raptor Wasm Godot on Firefox with Gecko Profiling"
    try-name: raptor-wasm-godot-firefox-profiling
    treeherder-symbol: Rap-Prof(godot)
+    run-on-projects: []
    max-run-time: 900
    mozharness:
        extra-options:
@ -435,6 +443,7 @@ raptor-wasm-godot-baseline-firefox:
    variants: ["fission"]
    try-name: raptor-wasm-godot-baseline-firefox
    treeherder-symbol: Rap(godot-b)
+    run-on-projects: []
    max-run-time:
        by-test-platform:
            .*-qr/.*: 2100
@ -448,6 +457,7 @@ raptor-wasm-godot-baseline-firefox-profiling:
    description: "Raptor Wasm Godot on Firefox with baseline JIT and Gecko Profiling"
    try-name: raptor-wasm-godot-baseline-firefox-profiling
    treeherder-symbol: Rap-Prof(godot-b)
+    run-on-projects: []
    max-run-time: 900
    mozharness:
        extra-options:
@ -459,6 +469,7 @@ raptor-wasm-godot-optimizing-firefox:
    variants: ["fission"]
    try-name: raptor-wasm-godot-optimizing-firefox
    treeherder-symbol: Rap(godot-i)
+    run-on-projects: []
    max-run-time:
        by-test-platform:
            .*-qr/.*: 2100
@ -472,6 +483,7 @@ raptor-wasm-godot-optimizing-firefox-profiling:
    description: "Raptor WASM godot on Firefox with optimizing JIT and Gecko Profiling"
    try-name: raptor-wasm-godot-optimizing-firefox-profiling
    treeherder-symbol: Rap-Prof(godot-i)
+    run-on-projects: []
    max-run-time: 900
    mozharness:
        extra-options:
--- a/testing/raptor/raptor/output.py
+++ b/testing/raptor/raptor/output.py
@ -859,6 +859,47 @@ class PerftestOutput(object):
        print(subtests)
        return subtests, vals

+    def parseWASMGodotOutput(self, test):
+        """
+        {u'wasm-godot': [
+            {
+              "name": "wasm-instantiate",
+              "time": 349
+            },{
+              "name": "engine-instantiate",
+              "time": 1263
+            ...
+            }]}
+        """
+        _subtests = {}
+        data = test["measurements"]["wasm-godot"]
+        print(data)
+        for page_cycle in data:
+            for item in page_cycle[0]:
+                # for each pagecycle, build a list of subtests and append all related replicates
+                sub = item["name"]
+                if sub not in _subtests:
+                    # subtest not added yet, first pagecycle, so add new one
+                    _subtests[sub] = {
+                        "unit": test["subtest_unit"],
+                        "alertThreshold": float(test["alert_threshold"]),
+                        "lowerIsBetter": test["subtest_lower_is_better"],
+                        "name": sub,
+                        "replicates": [],
+                    }
+                _subtests[sub]["replicates"].append(item["time"])
+
+        vals = []
+        subtests = []
+        names = list(_subtests)
+        names.sort(reverse=True)
+        for name in names:
+            _subtests[name]["value"] = filters.median(_subtests[name]["replicates"])
+            subtests.append(_subtests[name])
+            vals.append([_subtests[name]["value"], name])
+
+        return subtests, vals
+

 class RaptorOutput(PerftestOutput):
    """class for raptor output"""
@ -1224,47 +1265,6 @@ class RaptorOutput(PerftestOutput):

        return subtests, vals

-    def parseWASMGodotOutput(self, test):
-        """
-        {u'wasm-godot': [
-            {
-              "name": "wasm-instantiate",
-              "time": 349
-            },{
-              "name": "engine-instantiate",
-              "time": 1263
-            ...
-            }]}
-        """
-        _subtests = {}
-        data = test["measurements"]["wasm-godot"]
-        print(data)
-        for page_cycle in data:
-            for item in page_cycle[0]:
-                # for each pagecycle, build a list of subtests and append all related replicates
-                sub = item["name"]
-                if sub not in _subtests:
-                    # subtest not added yet, first pagecycle, so add new one
-                    _subtests[sub] = {
-                        "unit": test["subtest_unit"],
-                        "alertThreshold": float(test["alert_threshold"]),
-                        "lowerIsBetter": test["subtest_lower_is_better"],
-                        "name": sub,
-                        "replicates": [],
-                    }
-                _subtests[sub]["replicates"].append(item["time"])
-
-        vals = []
-        subtests = []
-        names = list(_subtests)
-        names.sort(reverse=True)
-        for name in names:
-            _subtests[name]["value"] = filters.median(_subtests[name]["replicates"])
-            subtests.append(_subtests[name])
-            vals.append([_subtests[name]["value"], name])
-
-        return subtests, vals
-
    def parseSunspiderOutput(self, test):
        _subtests = {}
        data = test["measurements"]["sunspider"]
@ -1562,6 +1562,8 @@ class BrowsertimeOutput(PerftestOutput):
                    subtests, vals = self.parseUnityWebGLOutput(test)
                if "webaudio" in test["measurements"]:
                    subtests, vals = self.parseWebaudioOutput(test)
+                if "wasm-godot" in test["measurements"]:
+                    subtests, vals = self.parseWASMGodotOutput(test)

                if subtests is None:
                    raise Exception("No benchmark metrics found in browsertime results")
--- a/testing/raptor/raptor/raptor.ini
+++ b/testing/raptor/raptor/raptor.ini
@ -53,3 +53,6 @@
 [include:tests/benchmarks/motionmark-htmlsuite.ini]
 [include:tests/benchmarks/stylebench.ini]
 [include:tests/benchmarks/webaudio.ini]
+[include:tests/benchmarks/wasm-godot.ini]
+[include:tests/benchmarks/wasm-godot-baseline.ini]
+[include:tests/benchmarks/wasm-godot-optimizing.ini]
--- a/testing/raptor/raptor/tests/benchmarks/wasm-godot-baseline.ini
+++ b/testing/raptor/raptor/tests/benchmarks/wasm-godot-baseline.ini
@ -0,0 +1,22 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Wasm-godot benchmark for firefox and chromium distributions
+
+[DEFAULT]
+type =  benchmark
+test_url = http://localhost:<port>/wasm-godot/index.html
+page_cycles = 5
+page_timeout = 120000
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+newtab_per_cycle = true
+gecko_profile_interval = 1
+gecko_profile_entries = 8000000
+
+[wasm-godot-baseline]
+apps = firefox
+preferences = {"javascript.options.wasm_baselinejit": true,
+               "javascript.options.wasm_optimizingjit": false}
--- a/testing/raptor/raptor/tests/benchmarks/wasm-godot-optimizing.ini
+++ b/testing/raptor/raptor/tests/benchmarks/wasm-godot-optimizing.ini
@ -0,0 +1,22 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Wasm-godot benchmark for firefox and chromium distributions
+
+[DEFAULT]
+type =  benchmark
+test_url = http://localhost:<port>/wasm-godot/index.html
+page_cycles = 5
+page_timeout = 120000
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+newtab_per_cycle = true
+gecko_profile_interval = 1
+gecko_profile_entries = 8000000
+
+[wasm-godot-optimizing]
+apps = firefox
+preferences = {"javascript.options.wasm_baselinejit": false,
+               "javascript.options.wasm_optimizingjit": true}
--- a/testing/raptor/raptor/tests/benchmarks/wasm-godot.ini
+++ b/testing/raptor/raptor/tests/benchmarks/wasm-godot.ini
@ -0,0 +1,20 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Wasm-godot benchmark for firefox and chromium distributions
+
+[DEFAULT]
+type =  benchmark
+test_url = http://localhost:<port>/wasm-godot/index.html
+page_cycles = 5
+page_timeout = 120000
+apps = firefox, chrome, chromium
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+newtab_per_cycle = true
+gecko_profile_interval = 1
+gecko_profile_entries = 8000000
+
+[wasm-godot]
--- a/third_party/webkit/PerformanceTests/wasm-godot/index.html
+++ b/third_party/webkit/PerformanceTests/wasm-godot/index.html
@ -377,6 +377,7 @@

 				_data = ['raptor-benchmark', 'wasm-godot', results];
 				window.postMessage(_data, '*');
+				window.sessionStorage.setItem('benchmark_results',  JSON.stringify(_data));
 			};

 			game.start(BASENAME + '.pck').then(() => {