Bug 1591466 - split raptor tests into tier-1 (high value) and tier-2 (lower value). r=sparky

split raptor tests into tier-1 (high value) and tier-2 (lower value)

Differential Revision: https://phabricator.services.mozilla.com/D78247

taskcluster/ci/config.yml

@@ -66,6 +66,19 @@ treeherder:
         'Rap-fennec': 'Raptor performance tests on Fennec'
         'Rap-fennec-1proc': 'Raptor performance tests on Fennec without e10s'
         'Rap-refbrow': 'Raptor performance tests on the reference browser'
+        'Rap-t2': 'Raptor performance tests on Firefox'
+        'Rap-t2-1proc': 'Raptor performance tests on Firefox without e10s'
+        'Rap-t2-fis': 'Raptor performance tests on Firefox with fission enabled'
+        'Rap-t2-Prof': 'Raptor performance tests on Firefox with Gecko Profiling'
+        'Rap-t2-Prof-1proc': 'Raptor performance tests on Firefox with Gecko Profiling and without e10s'
+        'Rap-t2-ChR': 'Raptor performance tests on Google Chrome Release'
+        'Rap-t2-Cr': 'Raptor performance tests on Google Chromium'
+        'Rap-t2-P': 'Raptor power tests on Firefox'
+        'Rap-t2-P-1proc': 'Raptor power tests on Firefox without e10s'
+        'Rap-t2-fenix': 'Raptor performance tests on Fenix'
+        'Rap-t2-fennec': 'Raptor performance tests on Fennec'
+        'Rap-t2-fennec-1proc': 'Raptor performance tests on Fennec without e10s'
+        'Rap-t2-refbrow': 'Raptor performance tests on the reference browser'
         'remote': 'Remote debugging protocol tests'
         'T': 'Talos performance tests'
         'T-1proc': 'Talos performance tests without e10s'

taskcluster/ci/test/raptor.yml

@@ -74,7 +74,7 @@ job-defaults:
                 by-test-platform:
                     windows10-64-ccov/.*: 3
                     linux64-ccov/.*: 3
-                    default: 1
+                    default: 2
             default: 2
     fission-tier: 2
     virtualization:
@@ -134,6 +134,10 @@ job-defaults:
 raptor-tp6:
     description: "Raptor (webextension) tp6"
     apps: ["firefox", "chrome", "chromium"]
+    tier:
+        by-app:
+            firefox: 1
+            default: 2
     variants:
         # Used for defining webrender and fission build variant tests,
         by-app:
@@ -159,6 +163,41 @@ raptor-tp6:
     raptor-test: tp6
     raptor-subtests:
         - amazon
+        - [google-mail, gmail]
+        - [slides, gslides]
+        - imgur
+        - tumblr
+        - twitch
+        - twitter
+
+
+raptor-tp6-t2:
+    description: "Raptor (webextension) tp6"
+    apps: ["firefox", "chrome", "chromium"]
+    variants:
+        # Used for defining webrender and fission build variant tests,
+        by-app:
+            firefox: ["fission"]
+            chrome: []
+            chromium: []
+            default: []
+    pageload:
+        by-app:
+            firefox: "both"
+            chrome: "both"
+            chromium: "both"
+            default: "cold"
+    limit-platforms:
+        by-app:
+            chrome:
+                - .*shippable[^-qr].*
+            chromium:
+                - .*shippable[^-qr].*
+            default: []
+    treeherder-symbol: Rap-t2(tp6)
+    max-run-time: 4000
+    raptor-test: tp6
+    raptor-subtests:
         - apple
         - [binast-instagram, inst-b]
         - bing
@@ -167,12 +206,9 @@ raptor-tp6:
         - [facebook-redesign, fb-r]
         - fandom
         - [docs, gdoc]
-        - [google-mail, gmail]
         - google
         - [sheets, gsheets]
-        - [slides, gslides]
         - imdb
-        - imgur
         - instagram
         - linkedin
         - microsoft
@@ -182,14 +218,11 @@ raptor-tp6:
         - paypal
         - pinterest
         - reddit
-        - twitch
-        - twitter
         - wikipedia
         - yahoo-mail
+        - yahoo-news
         - yandex
         - youtube
-        - tumblr
-        - yahoo-news
 
 raptor-speedometer-firefox:
     description: "Raptor Speedometer on Firefox"
@@ -205,7 +238,6 @@ raptor-speedometer-firefox-profiling:
     try-name: raptor-speedometer-firefox-profiling
     treeherder-symbol: Rap-Prof(sp)
     max-run-time: 900
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-speedometer
@@ -225,7 +257,6 @@ raptor-stylebench-firefox-profiling:
     try-name: raptor-stylebench-firefox-profiling
     treeherder-symbol: Rap-Prof(sb)
     max-run-time: 900
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-stylebench
@@ -244,7 +275,6 @@ raptor-motionmark-htmlsuite-firefox-profiling:
     description: "Raptor MotionMark HtmlSuite on Firefox with Gecko Profiling"
     try-name: raptor-motionmark-htmlsuite-firefox-profiling
     treeherder-symbol: Rap-Prof(mm-h)
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-motionmark-htmlsuite
@@ -255,6 +285,7 @@ raptor-motionmark-animometer-firefox:
     variants: ["fission"]
     try-name: raptor-motionmark-animometer-firefox
     treeherder-symbol: Rap(mm-a)
+    tier: 1
     mozharness:
         extra-options:
             - --test=raptor-motionmark-animometer
@@ -264,7 +295,6 @@ raptor-motionmark-animometer-firefox-profiling:
     try-name: raptor-motionmark-animometer-firefox-profiling
     treeherder-symbol: Rap-Prof(mm-a)
     max-run-time: 900
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-motionmark-animometer
@@ -275,6 +305,7 @@ raptor-webaudio-firefox:
     variants: ["fission"]
     try-name: raptor-webaudio-firefox
     treeherder-symbol: Rap(wa)
+    tier: 1
     mozharness:
         extra-options:
             - --test=raptor-webaudio
@@ -284,7 +315,6 @@ raptor-webaudio-firefox-profiling:
     try-name: raptor-webaudio-firefox
     treeherder-symbol: Rap-Prof(wa)
     max-run-time: 900
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-webaudio
@@ -304,7 +334,6 @@ raptor-sunspider-firefox-profiling:
     try-name: raptor-sunspider-firefox-profiling
     treeherder-symbol: Rap-Prof(ss)
     max-run-time: 900
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-sunspider
@@ -331,7 +360,6 @@ raptor-unity-webgl-firefox-profiling:
     description: "Raptor Unity WebGL on Firefox with Gecko Profiling"
     try-name: raptor-unity-webgl-firefox-profiling
     treeherder-symbol: Rap-Prof(ugl)
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-unity-webgl
@@ -350,7 +378,6 @@ raptor-wasm-misc-cranelift-firefox:
             (linux|windows|macos)(?!.*shippable).*: []
             default: ["mozilla-central", "try"]
     max-run-time: 1800
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-wasm-misc-cranelift
@@ -375,7 +402,6 @@ raptor-wasm-misc-firefox-profiling:
     try-name: raptor-wasm-misc-firefox-profiling
     treeherder-symbol: Rap-Prof(wm)
     max-run-time: 900
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-wasm-misc
@@ -401,7 +427,6 @@ raptor-wasm-misc-baseline-firefox-profiling:
     try-name: raptor-wasm-misc-baseline-firefox-profiling
     treeherder-symbol: Rap-Prof(wm-b)
     max-run-time: 900
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-wasm-misc-baseline
@@ -427,7 +452,6 @@ raptor-wasm-misc-ion-firefox-profiling:
     try-name: raptor-wasm-misc-ion-firefox-profiling
     treeherder-symbol: Rap-Prof(wm-i)
     max-run-time: 900
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-wasm-misc-ion
@@ -458,7 +482,6 @@ raptor-assorted-dom-firefox-profiling:
     try-name: raptor-assorted-dom-firefox-profiling
     treeherder-symbol: Rap-Prof(dom)
     max-run-time: 900
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-assorted-dom
@@ -486,7 +509,6 @@ raptor-wasm-godot-firefox-profiling:
     try-name: raptor-wasm-godot-firefox-profiling
     treeherder-symbol: Rap-Prof(godot)
     max-run-time: 900
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-wasm-godot
@@ -511,7 +533,6 @@ raptor-wasm-godot-baseline-firefox-profiling:
     try-name: raptor-wasm-godot-baseline-firefox-profiling
     treeherder-symbol: Rap-Prof(godot-b)
     max-run-time: 900
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-wasm-godot-baseline
@@ -531,7 +552,6 @@ raptor-wasm-godot-cranelift-firefox:
             .*-qr/.*: 2100
             .*-ref-hw-2017/.*: 2100
             default: 1500
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-wasm-godot-cranelift
@@ -541,7 +561,6 @@ raptor-wasm-godot-cranelift-firefox-profiling:
     try-name: raptor-wasm-godot-cranelift-firefox-profiling
     treeherder-symbol: Rap-Prof(godot-c)
     max-run-time: 900
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-wasm-godot-cranelift
@@ -566,7 +585,6 @@ raptor-wasm-godot-ion-firefox-profiling:
     try-name: raptor-wasm-godot-ion-firefox-profiling
     treeherder-symbol: Rap-Prof(godot-i)
     max-run-time: 900
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-wasm-godot-ion
@@ -583,7 +601,6 @@ raptor-youtube-playback-firefox:
             (linux|windows|macos)(?!.*shippable).*: []
             default: ["mozilla-central", "try"]
     max-run-time: 3000
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-youtube-playback
@@ -593,7 +610,6 @@ raptor-youtube-playback-firefox-profiling:
     try-name: raptor-youtube-playback-firefox-profiling
     treeherder-symbol: Rap-Prof(ytp)
     max-run-time: 4000
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-youtube-playback
@@ -646,7 +662,6 @@ raptor-youtube-playback-av1-sfr-firefox:
         by-test-platform:
             windows10-aarch64/opt: 3600
             default: 1800
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-youtube-playback-av1-sfr
@@ -665,7 +680,6 @@ raptor-youtube-playback-h264-sfr-firefox:
         by-test-platform:
             windows10-aarch64/opt: 3600
             default: 1800
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-youtube-playback-h264-sfr
@@ -684,7 +698,6 @@ raptor-youtube-playback-vp9-sfr-firefox:
         by-test-platform:
             windows10-aarch64/opt: 3600
             default: 1800
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-youtube-playback-vp9-sfr
@@ -703,7 +716,6 @@ raptor-youtube-playback-widevine-hfr-firefox:
         by-test-platform:
             windows10-aarch64/opt: 3600
             default: 2700
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-youtube-playback-widevine-hfr
@@ -722,7 +734,6 @@ raptor-youtube-playback-widevine-h264-sfr-firefox:
         by-test-platform:
             windows10-aarch64/opt: 3600
             default: 2700
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-youtube-playback-widevine-h264-sfr
@@ -741,7 +752,6 @@ raptor-youtube-playback-widevine-vp9-sfr-firefox:
         by-test-platform:
             windows10-aarch64/opt: 3600
             default: 2700
-    tier: 2
     mozharness:
         extra-options:
             - --test=raptor-youtube-playback-widevine-vp9-sfr

taskcluster/ci/test/test-sets.yml

@@ -77,6 +77,7 @@ talos-ref-hw:
 
 raptor-firefox:
     - raptor-tp6
+    - raptor-tp6-t2
     - raptor-speedometer-firefox
     - raptor-stylebench-firefox
     - raptor-motionmark-htmlsuite-firefox

taskcluster/taskgraph/transforms/raptor.py

@@ -31,6 +31,7 @@ raptor_description_schema = Schema({
     Optional('raptor-test'): text_type,
     Optional('raptor-subtests'): optionally_keyed_by(
         'app',
+        'tier',
         list
     ),
     Optional('activity'): optionally_keyed_by(
@@ -69,6 +70,7 @@ raptor_description_schema = Schema({
     Optional('tier'): optionally_keyed_by(
         'app',
         'raptor-test',
+        'raptor-subtests',
         test_description_schema['tier']
     ),
     Optional('run-visual-metrics'): optionally_keyed_by(