gecko-dev/taskcluster/ci/test/test-sets.yml

644 lines
18 KiB
YAML

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
---
# Each key in this file specifies a set of tests to run. Different test sets
# may, for example, be bound to different test platforms.
#
# Note that set does not depend on the tree; tree-dependent job selection
# should be performed in the target task selection phase of task-graph
# generation.
#
# A test set has a name, and a list of tests that it contains.
#
# Test names given here reference tests.yml.
### Feature-specific test groups
web-platform-tests:
- web-platform-tests
- web-platform-tests-crashtest
- web-platform-tests-print-reftest
- web-platform-tests-reftest
- web-platform-tests-wdspec
web-platform-tests-backlog:
- web-platform-tests-backlog
- web-platform-tests-reftest-backlog
web-platform-tests-wdspec-headless:
- web-platform-tests-wdspec-headless
mochitest-webgpu:
- mochitest-webgpu
talos:
- talos-bcv
- talos-chrome
- talos-damp
- talos-dromaeojs
- talos-g1
- talos-g3
- talos-g4
- talos-g5
- talos-other
- talos-sessionrestore-many-windows
- talos-svgr
- talos-tp5o
- talos-perf-reftest
- talos-perf-reftest-singletons
- talos-tabswitch
- talos-realworld-webextensions
- talos-webgl
# - talos-h1 Bug 1487031 - Disabled for not finding actionable regressions
# - talos-h2 Bug 1487031 - Disabled for not finding actionable regressions
talos-profiling:
- talos-bcv-profiling
- talos-chrome-profiling
# - talos-damp-profiling - Disabled b/c of Bug 1597734
- talos-dromaeojs-profiling
# - talos-flex-profiling - Disabled b/c of Bug 1597736
- talos-g1-profiling
- talos-g3-profiling
- talos-g4-profiling
- talos-g5-profiling
- talos-motionmark-profiling
- talos-other-profiling
- talos-sessionrestore-many-windows-profiling
- talos-perf-reftest-profiling
- talos-perf-reftest-singletons-profiling
- talos-svgr-profiling
- talos-tp5o-profiling
- talos-tabswitch-profiling
- talos-realworld-webextensions-profiling
- talos-webgl-profiling
talos-ref-hw:
- talos-g4
- talos-webgl
raptor-firefox:
- raptor-tp6
- raptor-speedometer-firefox
- raptor-stylebench-firefox
- raptor-motionmark-htmlsuite-firefox
- raptor-motionmark-animometer-firefox
- raptor-webaudio-firefox
- raptor-sunspider-firefox
- raptor-wasm-godot-firefox
- raptor-youtube-playback-firefox
- raptor-youtube-playback-av1-sfr-firefox
- raptor-youtube-playback-h264-sfr-firefox
- raptor-youtube-playback-h264-firefox
- raptor-youtube-playback-vp9-sfr-firefox
- raptor-youtube-playback-widevine-hfr-firefox
- raptor-youtube-playback-widevine-h264-sfr-firefox
- raptor-youtube-playback-widevine-vp9-sfr-firefox
- raptor-ares6-firefox
- raptor-jetstream2-firefox
raptor-chrome:
- raptor-speedometer-chrome
- raptor-stylebench-chrome
- raptor-motionmark-htmlsuite-chrome
- raptor-motionmark-animometer-chrome
- raptor-webaudio-chrome
- raptor-sunspider-chrome
- raptor-wasm-godot-chrome
- raptor-ares6-chrome
- raptor-jetstream2-chrome
- raptor-youtube-playback-chrome
- raptor-youtube-playback-av1-sfr-chrome
- raptor-youtube-playback-h264-sfr-chrome
- raptor-youtube-playback-h264-chrome
- raptor-youtube-playback-vp9-sfr-chrome
- raptor-youtube-playback-widevine-hfr-chrome
- raptor-youtube-playback-widevine-h264-sfr-chrome
- raptor-youtube-playback-widevine-vp9-sfr-chrome
raptor-profiling:
- raptor-speedometer-firefox-profiling
- raptor-stylebench-firefox-profiling
- raptor-motionmark-htmlsuite-firefox-profiling
- raptor-motionmark-animometer-firefox-profiling
- raptor-webaudio-firefox-profiling
- raptor-sunspider-firefox-profiling
- raptor-wasm-godot-firefox-profiling
- raptor-ares6-firefox-profiling
- raptor-jetstream2-firefox-profiling
- raptor-youtube-playback-firefox-profiling
raptor-chromium:
- raptor-speedometer-chromium
- raptor-stylebench-chromium
- raptor-motionmark-htmlsuite-chromium
- raptor-motionmark-animometer-chromium
- raptor-webaudio-chromium
- raptor-sunspider-chromium
- raptor-wasm-godot-chromium
- raptor-ares6-chromium
- raptor-jetstream2-chromium
# Fetch tasks are only supported on Linux for now,
# so these need to be separate sets.
raptor-fetch-firefox:
- raptor-unity-webgl-firefox
- raptor-wasm-misc-firefox
- raptor-wasm-misc-baseline-firefox
- raptor-wasm-misc-ion-firefox
- raptor-wasm-misc-cranelift-firefox
- raptor-wasm-godot-baseline-firefox
- raptor-wasm-godot-ion-firefox
- raptor-wasm-godot-cranelift-firefox
- raptor-assorted-dom-firefox
raptor-fetch-profiling:
- raptor-unity-webgl-firefox-profiling
- raptor-wasm-misc-firefox-profiling
- raptor-wasm-misc-baseline-firefox-profiling
- raptor-wasm-misc-ion-firefox-profiling
- raptor-wasm-godot-baseline-firefox-profiling
- raptor-wasm-godot-ion-firefox-profiling
- raptor-wasm-godot-cranelift-firefox-profiling
- raptor-assorted-dom-firefox-profiling
raptor-fetch-geckoview:
- raptor-unity-webgl-geckoview
- raptor-unity-webgl-refbrow
raptor-fetch-chromium:
- raptor-unity-webgl-chromium
- raptor-wasm-misc-chromium
- raptor-assorted-dom-chromium
- raptor-jetstream2-chromium
browsertime:
- browsertime-tp6
- browsertime-speedometer
- browsertime-ares6
- browsertime-tp6-profiling
# Raptor desktop power tests
# (requires Intel Power Gadget to be pre-installed)
raptor-firefox-power:
- raptor-youtube-playback-v9-power-firefox
- raptor-youtube-playback-h264-power-firefox
awsy:
- awsy
- awsy-base
- awsy-dmd
- awsy-base-dmd
- awsy-tp6
ccov-code-coverage-tests:
- test-coverage
- test-coverage-wpt
marionette-gpu-tests:
- marionette-gpu
mochitest-headless:
- mochitest-plain-headless
mochitest-valgrind:
- mochitest-valgrind
marionette-headless:
- marionette-headless
desktop-screenshot-capture:
- browser-screenshots
### Specific platform test sets
# Linux
linux32-tests:
- web-platform-tests
linux1804-tests:
- cppunit
- crashtest
- firefox-ui-functional-local
- firefox-ui-functional-remote
- gtest
- jittest
- jsreftest
- marionette
- mochitest-a11y
- mochitest-browser-chrome
- mochitest-chrome
- mochitest-chrome-gpu
- mochitest-devtools-chrome
- mochitest-media
- mochitest-plain
- mochitest-plain-gpu
- mochitest-remote
- mochitest-webgl1-core
- mochitest-webgl1-ext
- mochitest-webgl2-core
- mochitest-webgl2-ext
- reftest
- reftest-no-accel
- telemetry-tests-client
- test-verify
- test-verify-gpu
- test-verify-wpt
- xpcshell
linux1804-qr-tests:
- cppunit
- crashtest
- gtest
- jittest
- jsreftest
- mochitest-a11y
- mochitest-chrome
- mochitest-chrome-gpu
- mochitest-media
- mochitest-plain
- mochitest-plain-gpu
- mochitest-remote
- mochitest-webgl1-core
- mochitest-webgl1-ext
- mochitest-webgl2-core
- mochitest-webgl2-ext
- reftest
- xpcshell
linux-qr-smoketests:
- crashtest
- reftest
linux-talos-flex:
- talos-flex
# Windows
windows-qr-tests:
- crashtest
- mochitest-chrome-gpu
- mochitest-browser-chrome
- mochitest-plain
- mochitest-plain-gpu
- mochitest-media
- mochitest-webgl1-core
- mochitest-webgl2-core
- reftest
windows-reftest-gpu:
- reftest-gpu
windows-tests:
- cppunit
- crashtest
- firefox-ui-functional-local
- firefox-ui-functional-remote
- gtest
- jittest
- jsreftest
- marionette
- mochitest-a11y
- mochitest-browser-chrome
- mochitest-chrome
- mochitest-chrome-gpu
- mochitest-devtools-chrome
- mochitest-media
- mochitest-plain
- mochitest-plain-gpu
- mochitest-remote
- mochitest-webgl1-core
- mochitest-webgl1-ext
- mochitest-webgl2-core
- mochitest-webgl2-ext
- reftest
- reftest-no-accel
- telemetry-tests-client
- test-verify
- test-verify-gpu
- test-verify-wpt
- xpcshell
mingwclang-tests-debug:
- cppunit
- firefox-ui-functional-local
- firefox-ui-functional-remote
- mochitest-a11y
- mochitest-chrome-gpu
- mochitest-plain-gpu
- mochitest-webgl1-core
- mochitest-webgl1-ext
- mochitest-webgl2-core
- mochitest-webgl2-ext
- mochitest-webgpu
- reftest
- telemetry-tests-client
# The following is the same as the above block, minus reftests.
# Reftests were disabled due to frequent hangs; see bug 1642719
# for re-enabling.
mingwclang-win7-tests-debug:
- cppunit
- firefox-ui-functional-local
- firefox-ui-functional-remote
- mochitest-a11y
- mochitest-chrome-gpu
- mochitest-plain-gpu
- mochitest-webgl1-core
- mochitest-webgl1-ext
- mochitest-webgl2-core
- mochitest-webgl2-ext
- mochitest-webgpu
# - reftest - disabled, see bug 1642719
- telemetry-tests-client
mingwclang-tests-opt:
- cppunit
- mochitest-chrome-gpu
- mochitest-plain-gpu
windows-aarch64-tests:
- crashtest
- mochitest-media
- reftest
windows-talos:
- talos-bcv
- talos-chrome
- talos-damp
- talos-dromaeojs
- talos-g1
- talos-g4
- talos-g5
- talos-other
- talos-sessionrestore-many-windows
- talos-perf-reftest
- talos-perf-reftest-singletons
- talos-svgr
- talos-tp5o
- talos-xperf
- talos-tabswitch
- talos-realworld-webextensions
- talos-webgl
# - talos-h1 Bug 1487031 - Disabled for not finding actionable regressions
# macOS
macosx1014-64-tests:
- cppunit
- crashtest
- firefox-ui-functional-local
- firefox-ui-functional-remote
- gtest
- jittest
- jsreftest
- marionette
- mochitest-a11y
- mochitest-browser-chrome
- mochitest-chrome
- mochitest-chrome-gpu
- mochitest-devtools-chrome
- mochitest-plain
- mochitest-plain-gpu
- mochitest-media
- mochitest-remote
- mochitest-webgl1-core
- mochitest-webgl1-ext
- mochitest-webgl2-core
- mochitest-webgl2-ext
- mochitest-webgpu
- reftest
- telemetry-tests-client
- test-verify
- test-verify-gpu
- test-verify-wpt
- xpcshell
macosx1014-64-qr-tests:
- crashtest
- reftest
- web-platform-tests-reftest
macosx64-talos:
- talos-bcv
- talos-chrome
- talos-damp
- talos-dromaeojs
- talos-g1
- talos-g4
- talos-g5
- talos-other
- talos-sessionrestore-many-windows
- talos-svgr
- talos-tp5o
- talos-perf-reftest
- talos-perf-reftest-singletons
- talos-realworld-webextensions
- talos-webgl
# - talos-tabswitch # Bug 1453007 times out
# - talos-h1 # too long to unpack profile- Bug 1442893
# Android
android-x86_64-tests:
- cppunit
- crashtest
- geckoview-junit
- gtest
- mochitest-media
- mochitest-plain
- mochitest-plain-gpu
# - mochitest-webgl1-core # Harness issues?
# - mochitest-webgl1-ext
# - mochitest-webgl2-core # Emulator not creating ES3?
# - mochitest-webgl2-ext
- reftest
- test-verify
- web-platform-tests
- web-platform-tests-crashtest
- web-platform-tests-reftest
- xpcshell
android-x86_64-qr-tests:
- crashtest
- reftest
- mochitest-media
- mochitest-plain
- mochitest-plain-gpu
android-hw-aarch64-opt-unittests:
- jittest
- mochitest-media
- mochitest-webgl1-core
- mochitest-webgl1-ext
- mochitest-webgl2-core
android-hw-arm7-pgo-unittests:
- jittest
- jsreftest
android-hw-aarch64-pgo-unittests:
- jittest
- mochitest-media
- mochitest-webgl1-core
- mochitest-webgl1-ext
- mochitest-webgl2-core
android-hw-arm7-debug-unittests:
- jittest
- jsreftest
android-hw-aarch64-debug-unittests:
- jittest
- mochitest-media
- mochitest-webgl1-core
- mochitest-webgl1-ext
- mochitest-webgl2-core
android-hw-aarch64-qr:
- crashtest-qr
- reftest-qr
android-hw-arm7-raptor:
- raptor-speedometer-geckoview
- raptor-speedometer-refbrow
- raptor-speedometer-fenix
- raptor-speedometer-fennec68
- raptor-youtube-playback-fenix
- raptor-youtube-playback-av1-sfr-fenix
- raptor-youtube-playback-h264-sfr-fenix
- raptor-youtube-playback-h264-fenix
- raptor-youtube-playback-vp9-sfr-fenix
- raptor-youtube-playback-widevine-hfr-fenix
- raptor-youtube-playback-widevine-h264-sfr-fenix
- raptor-youtube-playback-widevine-vp9-sfr-fenix
- raptor-youtube-playback-geckoview
- raptor-youtube-playback-av1-sfr-geckoview
- raptor-youtube-playback-h264-sfr-geckoview
- raptor-youtube-playback-h264-geckoview
- raptor-youtube-playback-vp9-sfr-geckoview
- raptor-youtube-playback-widevine-hfr-geckoview
- raptor-youtube-playback-widevine-h264-sfr-geckoview
- raptor-youtube-playback-widevine-vp9-sfr-geckoview
- raptor-youtube-playback-fennec68
- raptor-youtube-playback-av1-sfr-fennec68
- raptor-youtube-playback-h264-sfr-fennec68
- raptor-youtube-playback-h264-fennec68
- raptor-youtube-playback-vp9-sfr-fennec68
- raptor-youtube-playback-widevine-hfr-fennec68
- raptor-youtube-playback-widevine-h264-sfr-fennec68
- raptor-youtube-playback-widevine-vp9-sfr-fennec68
android-hw-aarch64-raptor:
- raptor-speedometer-geckoview
- raptor-speedometer-refbrow
- raptor-speedometer-fenix
- raptor-speedometer-fennec68
- raptor-youtube-playback-fenix
- raptor-youtube-playback-av1-sfr-fenix
- raptor-youtube-playback-h264-sfr-fenix
- raptor-youtube-playback-h264-fenix
- raptor-youtube-playback-vp9-sfr-fenix
- raptor-youtube-playback-widevine-hfr-fenix
- raptor-youtube-playback-widevine-h264-sfr-fenix
- raptor-youtube-playback-widevine-vp9-sfr-fenix
- raptor-youtube-playback-geckoview
- raptor-youtube-playback-av1-sfr-geckoview
- raptor-youtube-playback-h264-sfr-geckoview
- raptor-youtube-playback-h264-geckoview
- raptor-youtube-playback-vp9-sfr-geckoview
- raptor-youtube-playback-widevine-hfr-geckoview
- raptor-youtube-playback-widevine-h264-sfr-geckoview
- raptor-youtube-playback-widevine-vp9-sfr-geckoview
- raptor-youtube-playback-fennec68
- raptor-youtube-playback-av1-sfr-fennec68
- raptor-youtube-playback-h264-sfr-fennec68
- raptor-youtube-playback-h264-fennec68
- raptor-youtube-playback-vp9-sfr-fennec68
- raptor-youtube-playback-widevine-hfr-fennec68
- raptor-youtube-playback-widevine-h264-sfr-fennec68
- raptor-youtube-playback-widevine-vp9-sfr-fennec68
android-hw-arm7-raptor-power:
- raptor-speedometer-geckoview-power
- raptor-speedometer-fenix-power
- raptor-speedometer-fennec68-power
- raptor-scn-power-idle-fenix
- raptor-scn-power-idle-fennec68
- raptor-scn-power-idle-geckoview
- raptor-scn-power-idle-refbrow
- raptor-scn-power-idle-bg-fenix
- raptor-scn-power-idle-bg-fennec68
- raptor-scn-power-idle-bg-geckoview
- raptor-scn-power-idle-bg-refbrow
android-hw-arm7-raptor-cpu-memory:
- raptor-speedometer-geckoview-cpu-memory
- raptor-scn-cpu-memory-idle-fenix
- raptor-scn-cpu-memory-idle-geckoview
- raptor-scn-cpu-memory-idle-refbrow
- raptor-scn-cpu-memory-idle-bg-fenix
- raptor-scn-cpu-memory-idle-bg-geckoview
- raptor-scn-cpu-memory-idle-bg-refbrow
android-hw-arm7-raptor-cpu-memory-power:
- raptor-speedometer-geckoview-cpu-memory-power
- raptor-scn-cpu-memory-power-idle-fenix
- raptor-scn-cpu-memory-power-idle-geckoview
- raptor-scn-cpu-memory-power-idle-refbrow
- raptor-scn-cpu-memory-power-idle-bg-fenix
- raptor-scn-cpu-memory-power-idle-bg-geckoview
- raptor-scn-cpu-memory-power-idle-bg-refbrow
android-hw-aarch64-raptor-power:
- raptor-speedometer-geckoview-power
- raptor-speedometer-fenix-power
- raptor-speedometer-fennec68-power
- raptor-scn-power-idle-fenix
- raptor-scn-power-idle-fennec68
- raptor-scn-power-idle-geckoview
- raptor-scn-power-idle-refbrow
- raptor-scn-power-idle-bg-fenix
- raptor-scn-power-idle-bg-fennec68
- raptor-scn-power-idle-bg-geckoview
- raptor-scn-power-idle-bg-refbrow
android-hw-aarch64-raptor-cpu-memory:
- raptor-speedometer-geckoview-cpu-memory
- raptor-scn-cpu-memory-idle-fenix
- raptor-scn-cpu-memory-idle-geckoview
- raptor-scn-cpu-memory-idle-refbrow
- raptor-scn-cpu-memory-idle-bg-fenix
- raptor-scn-cpu-memory-idle-bg-geckoview
- raptor-scn-cpu-memory-idle-bg-refbrow
android-hw-aarch64-raptor-cpu-memory-power:
- raptor-speedometer-geckoview-cpu-memory-power
- raptor-scn-cpu-memory-power-idle-fenix
- raptor-scn-cpu-memory-power-idle-geckoview
- raptor-scn-cpu-memory-power-idle-refbrow
- raptor-scn-cpu-memory-power-idle-bg-fenix
- raptor-scn-cpu-memory-power-idle-bg-geckoview
- raptor-scn-cpu-memory-power-idle-bg-refbrow
android-hw-aarch64-browsertime:
- browsertime-tp6m
- browsertime-tp6m-live
- browsertime-tp6m-profiling
- browsertime-speedometer-mobile
- browsertime-youtube-playback-mobile
android-hw-arm7-browsertime:
- browsertime-tp6m
- browsertime-tp6m-live
- browsertime-tp6m-profiling
- browsertime-speedometer-mobile
- browsertime-youtube-playback-mobile
linux-tsan-tests:
- mochitest-plain
- xpcshell