Bug 1806035 - Enable Windows Speedometer 3 tests. r=perftest-reviewers,kshampur

This patch updates the S3 commit used, fixes an issue with local Windows benchmark runs, and enables the windows S3 tests in CI.

Differential Revision: https://phabricator.services.mozilla.com/D166323

taskcluster/ci/test/browsertime-desktop.yml

@@ -316,9 +316,8 @@ browsertime-benchmark:
                 by-app:
                     firefox:
                         by-test-platform:
-                            windows.*: []
-                            linux.*64.*shippable-qr/.*: [trunk, mozilla-release]
-                            mac.*64.*shippable-qr/.*: [trunk, mozilla-release]
+                            windows.*-32.*: []
+                            .*64.*shippable-qr/.*: [trunk, mozilla-release]
                             default: []
                     default: []
             unity-webgl:

taskcluster/gecko_taskgraph/target_tasks.py

@@ -750,8 +750,6 @@ def target_tasks_general_perf_testing(full_task_graph, parameters, graph_config)
         if "android" not in platform:
             # Select some browsertime tasks as desktop smoke-tests
             if "browsertime" in try_name:
-                if "speedometer3" in try_name and "windows" in platform:
-                    return False
                 if "chrome" in try_name:
                     if "tp6" in try_name and "macosx1014" in platform:
                         return False
@@ -768,6 +766,7 @@ def target_tasks_general_perf_testing(full_task_graph, parameters, graph_config)
                     if "speedometer" in try_name:
                         return True
                 if "safari" and "benchmark" in try_name:
+                    # Speedometer 3 is broken on Safari, see bug 1802922
                     if "speedometer3" in try_name:
                         return False
                     return True

testing/perfdocs/generated/raptor.rst

@@ -1964,7 +1964,7 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
    * **page timeout**: 180000
    * **repository**: https://github.com/camillobruni/Speedometer
    * **repository branch**: 2022-12-08_interactive_runner
-   * **repository revision**: 9d1643f8038c3931c6bb637472e970130ed4e4b6
+   * **repository revision**: 9e2d82c75a66cae636db1888e916276075a137b8
    * **submetric summary method**: median
    * **subtest lower is better**: true
    * **subtest name filters**: tests/,s3/
@@ -2143,9 +2143,9 @@ Standard benchmarks are third-party tests (i.e. Speedometer) that we have integr
         - ❌
         - ❌
       * - **browsertime-benchmark-firefox-speedometer3**
-        - ❌
-        - ❌
-        - ❌
+        - ✅
+        - ✅
+        - ✅
         - ❌
 
 

testing/raptor/browsertime/speedometer3.js

@@ -41,12 +41,26 @@ module.exports = async function(context, commands) {
             window.testDone = true;
           }
         };
-        const runner = new BenchmarkRunner(window.Suites, benchmarkClient);
-        console.log("created");
-        console.log(Suites);
-        const iterationCount = ${speedometer_iterations};
-        runner.runMultipleIterations(iterationCount);
-        console.log("running");
+        window.Suites.forEach((el) => {
+          if (window.navigator.platform.toLowerCase().includes("win")) {
+            if (
+                el.name == "React-TodoMVC" ||
+                el.name == "React-Redux-TodoMVC" ||
+                el.name == "EmberJS-TodoMVC"
+            ) {
+              el.disabled = true;
+            } else {
+              el.disabled = false;
+            }
+          } else {
+            el.disabled = false;
+          }
+        });
+        // BenchmarkRunner is overriden as the InteractiveBenchmarkRunner
+        const runner = new BenchmarkRunner(window.Suites, ${speedometer_iterations});
+        runner._client = benchmarkClient;
+
+        runner.runSuites();
     `);
 
     let data_exists = false;

testing/raptor/raptor/benchmark.py

@@ -138,7 +138,7 @@ class Benchmark(object):
                 os.symlink(benchmark_path, dest)
         else:
             # Clobber the benchmark in case a recent update removed any files.
-            mozfile.remove(dest)
+            mozfile.remove(str(dest.resolve()))
             shutil.copytree(benchmark_path, dest)
 
         if any(path.is_file() for path in benchmark_path.iterdir()):

testing/raptor/raptor/tests/benchmarks/speedometer-desktop.ini

@@ -27,7 +27,7 @@ custom_data = true
 owner = Performance Team
 repository = https://github.com/camillobruni/Speedometer
 repository_branch = 2022-12-08_interactive_runner
-repository_revision = 9d1643f8038c3931c6bb637472e970130ed4e4b6
+repository_revision = 9e2d82c75a66cae636db1888e916276075a137b8
 submetric_summary_method = median
 subtest_name_filters = tests/,s3/
 test_script = speedometer3.js