Bug 1661545 - Add a geckoview and desktop live site test on autoland. r=perftest-reviewers,Bebe

Differential Revision: https://phabricator.services.mozilla.com/D88680

taskcluster/ci/config.yml

@@ -43,6 +43,7 @@ treeherder:
         'Btime-Prof-fennec-1proc': 'Browsertime performance tests on Fennec without e10s with Gecko Profiling'
         'Btime-Prof-refbrow': 'Browsertime performance tests on the reference browser with Gecko Profiling'
         'Btime-wr': 'Browsertime performance tests on Android with webrender enabled'
+        'Btime-live-wr': 'Browsertime performance tests on Android with webrender enabled'
         'Fetch': 'Fetch and store content'
         'Fxfn-l': 'Firefox functional tests (local)'
         'Fxfn-l-1proc': 'Firefox functional tests (local) without e10s'
@@ -67,6 +68,7 @@ treeherder:
         'R-fis': 'Reftests with fission enabled'
         'R-swr': 'Reftests with software webrender enabled'
         'Rap': 'Raptor performance tests on Firefox'
+        'Rap-live': 'Raptor performance tests on Firefox with live sites'
         'Rap-1proc': 'Raptor performance tests on Firefox without e10s'
         'Rap-fis': 'Raptor performance tests on Firefox with fission enabled'
         'Rap-Prof': 'Raptor performance tests on Firefox with Gecko Profiling'

taskcluster/ci/test/browsertime-mobile.yml

@@ -252,49 +252,60 @@ browsertime-tp6m-live:
     description: "Raptor (browsertime) tp6 live site page-load tests on android"
     raptor-test: tp6m
     raptor-subtests:
-        - allrecipes
-        - amazon
-        - [amazon-search, amazon-s]
-        - bbc
-        - bing
-        - [bing-search-restaurants, bing-s-r]
-        - booking
-        - booking-sf
-        - cnn
-        - [cnn-ampstories, cnn-amp]
-        - discord
-        - [ebay-kleinanzeigen, ebay-k]
-        - [ebay-kleinanzeigen-search, ebay-k-s]
-        - espn
-        - expedia
-        - facebook
-        - [facebook-cristiano, fb-cris]
-        - [fashionbeans, fashbea]
-        - google
-        - [google-accounts, gacc]
-        - [google-maps, gmaps]
-        - [google-search-restaurants, gsearch-r]
-        - instagram
-        - imdb
-        - [imdb-firefox, imdb-f]
-        - jianshu
-        - [medium-article, medium]
-        - [microsoft-support, micros-sup]
-        - nytimes
-        - [people-article, people]
-        - reddit
-        - [reddit-thread, reddit-thr]
-        - rumble-fox
-        - [stackoverflow, stacko]
-        - [stackoverflow-question, stack-q]
-        - [urbandictionary-define, urbict-def]
-        - web-de
-        - [wikia-marvel, wikia-m]
-        - wikipedia
-        - youtube
-        - [youtube-watch, youtube-w]
-    apps: ["fenix", "chrome-m"]
-    run-on-projects: []
+        by-app:
+            geckoview:
+                - [cnn-ampstories, cnn-amp]
+            default:
+                - allrecipes
+                - amazon
+                - [amazon-search, amazon-s]
+                - bbc
+                - bing
+                - [bing-search-restaurants, bing-s-r]
+                - booking
+                - booking-sf
+                - cnn
+                - [cnn-ampstories, cnn-amp]
+                - discord
+                - [ebay-kleinanzeigen, ebay-k]
+                - [ebay-kleinanzeigen-search, ebay-k-s]
+                - espn
+                - expedia
+                - facebook
+                - [facebook-cristiano, fb-cris]
+                - [fashionbeans, fashbea]
+                - google
+                - [google-accounts, gacc]
+                - [google-maps, gmaps]
+                - [google-search-restaurants, gsearch-r]
+                - instagram
+                - imdb
+                - [imdb-firefox, imdb-f]
+                - jianshu
+                - [medium-article, medium]
+                - [microsoft-support, micros-sup]
+                - nytimes
+                - [people-article, people]
+                - reddit
+                - [reddit-thread, reddit-thr]
+                - rumble-fox
+                - [stackoverflow, stacko]
+                - [stackoverflow-question, stack-q]
+                - [urbandictionary-define, urbict-def]
+                - web-de
+                - [wikia-marvel, wikia-m]
+                - wikipedia
+                - youtube
+                - [youtube-watch, youtube-w]
+    apps: ["fenix", "chrome-m", "geckoview"]
+    run-on-projects:
+        by-app:
+            geckoview:
+                by-test-platform:
+                    android-hw-g5-7-0-arm7-api-16-shippable/opt: ["autoland"]
+                    android-hw-p2-8-0-android-aarch64-shippable/opt: ["autoland"]
+                    default: []
+            default: []
     pageload: cold
     run-visual-metrics: true
     treeherder-symbol: Btime-live(tp6m)

taskcluster/ci/test/raptor.yml

@@ -119,7 +119,6 @@ job-defaults:
                             - linux64-chromium
                 default: []
 
-
 raptor-tp6:
     description: "Raptor (webextension) tp6"
     apps: ["firefox", "chrome", "chromium"]
@@ -196,6 +195,32 @@ raptor-tp6:
                     default: 2
             default: 2
 
+raptor-tp6-live:
+    description: "Raptor (webextension) tp6 on live-sites"
+    apps: ["firefox"]
+    variants: []
+    pageload: "both"
+    treeherder-symbol: Rap-live(tp6)
+    raptor-test: tp6
+    raptor-subtests:
+        - [cnn-ampstories, cnn-amp]
+    run-on-projects:
+        by-pageload:
+            warm:
+                by-test-platform:
+                    (linux|windows|macos)(?!.*shippable).*: []
+                    default: ["mozilla-central"]
+            cold:
+                by-test-platform:
+                    macosx.*64(?:-qr)?/opt: ["mozilla-central"]
+                    (linux|windows|macos)(?!.*shippable).*: []
+                    default: ["mozilla-central", "autoland"]
+    tier: 2
+    mozharness:
+        extra-options:
+            - --no-conditioned-profile
+            - --live-sites
+
 raptor-speedometer-firefox:
     description: "Raptor Speedometer on Firefox"
     variants: ["fission"]

taskcluster/ci/test/test-sets.yml

@@ -80,6 +80,7 @@ talos-ref-hw:
 
 raptor-firefox:
     - raptor-tp6
+    - raptor-tp6-live
     - raptor-speedometer-firefox
     - raptor-stylebench-firefox
     - raptor-motionmark-htmlsuite-firefox

testing/perfdocs/generated/raptor.rst

@@ -38,6 +38,7 @@ Tests for page-load performance. The links direct to the actual websites that ar
 * `raptor-tp6-amazon (WX, FF, CH, CU) <https://www.amazon.com/s?k=laptop&ref=nb_sb_noss_1>`__
 * `raptor-tp6-apple (WX, FF, CH, CU) <https://www.apple.com/macbook-pro/>`__
 * `raptor-tp6-bing (WX, FF, CH, CU) <https://www.bing.com/search?q=barack+obama>`__
+* `raptor-tp6-cnn-ampstories (WX, FF) <https://cnn.com/ampstories/us/why-hurricane-michael-is-a-monster-unlike-any-other>`__
 * `raptor-tp6-docs (WX, FF, CH, CU) <https://docs.google.com/document/d/1US-07msg12slQtI_xchzYxcKlTs6Fp7WqIc6W5GK5M8/edit?usp=sharing>`__
 * `raptor-tp6-ebay (WX, FF, CH, CU) <https://www.ebay.com/>`__
 * `raptor-tp6-facebook (WX, CH, CU) <https://www.facebook.com>`__

testing/raptor/raptor/output.py

@@ -695,7 +695,7 @@ class RaptorOutput(PerftestOutput):
             suite = {
                 "name": test["name"],
                 "type": test["type"],
-                "tags": [],
+                "tags": test.get("tags", []),
                 "extraOptions": test["extra_options"],
                 "subtests": subtests,
                 "lowerIsBetter": test["lower_is_better"],
@@ -1470,6 +1470,7 @@ class BrowsertimeOutput(PerftestOutput):
                     "name": test["name"],
                     "type": test["type"],
                     "extraOptions": extra_options,
+                    "tags": test.get("tags", extra_options),
                     "lowerIsBetter": test["lower_is_better"],
                     "unit": test["unit"],
                     "alertThreshold": float(test["alert_threshold"]),

testing/raptor/raptor/perfdocs/config.yml

@@ -10,6 +10,7 @@ suites:
         description: "Tests for page-load performance. The links direct to the actual websites that are being tested. (WX: WebExtension, BT: Browsertime, FF: Firefox, CH: Chrome, CU: Chromium)"
         tests:
             raptor-tp6-amazon: "WX, FF, CH, CU"
+            raptor-tp6-cnn-ampstories: "WX, FF"
             raptor-tp6-facebook: "WX, CH, CU"
             raptor-tp6-google: "WX, FF, CH, CU"
             raptor-tp6-youtube: "WX, FF, CH, CU"

testing/raptor/raptor/results.py

@@ -241,6 +241,9 @@ class RaptorResultsHandler(PerftestResultsHandler):
             (self.fission_enabled, "fission"),
             (self.webrender_enabled, "webrender"),
         ]))
+        if self.live_sites:
+            new_result_json.setdefault("tags", []).append("live")
+            new_result_json["extra_options"].append("live")
         self.results.append(new_result_json)
 
     def summarize_and_output(self, test_config, tests, test_names):

testing/raptor/raptor/tests/tp6/desktop-tests.ini

@@ -26,6 +26,7 @@ alert_on = fcp, loadtime
 [include:desktop/amazon.ini]
 [include:desktop/apple.ini]
 [include:desktop/bing-search.ini]
+[include:desktop/cnn-ampstories.ini]
 [include:desktop/ebay.ini]
 [include:desktop/facebook.ini]
 [include:desktop/facebook-redesign.ini]

testing/raptor/raptor/tests/tp6/desktop/cnn-ampstories.ini

@@ -0,0 +1,14 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6-cnn-ampstories
+
+[DEFAULT]
+test_url = https://cnn.com/ampstories/us/why-hurricane-michael-is-a-monster-unlike-any-other
+alert_threshold = 5.0
+
+[raptor-tp6-cnn-ampstories-firefox]
+apps = firefox
+measure = fnbpaint, fcp, dcf, loadtime
+alert_on = fnbpaint, fcp, dcf, loadtime

testing/raptor/raptor/webextension/base.py

@@ -38,15 +38,7 @@ class WebExtension(Perftest):
         self.using_condprof = self.config.get("using_condprof", True)
 
         # set up the results handler
-        self.results_handler = RaptorResultsHandler(
-            gecko_profile=self.config.get("gecko_profile"),
-            power_test=self.config.get("power_test"),
-            cpu_test=self.config.get("cpu_test"),
-            memory_test=self.config.get("memory_test"),
-            no_conditioned_profile=self.config["no_conditioned_profile"],
-            extra_prefs=self.config.get("extra_prefs"),
-            enable_webrender=self.config["enable_webrender"],
-        )
+        self.results_handler = RaptorResultsHandler(**self.config)
         browser_name, browser_version = self.get_browser_meta()
         self.results_handler.add_browser_meta(self.config["app"], browser_version)