Bug 1545175 - Raptor test definitions and taskcluster configs for cold page-load tests on Fenix; r=Bebe,jmaher

Differential Revision: https://phabricator.services.mozilla.com/D27964 --HG-- extra : moz-landing-system : lando
2024-10-20 16:55:40 +00:00 · 2019-04-18 19:34:10 +00:00 · 2019-04-18 19:34:10 +00:00 · 1105b8f744
commit 1105b8f744
parent 4885178299
17 changed files with 697 additions and 6 deletions
--- a/taskcluster/ci/test/raptor.yml
+++ b/taskcluster/ci/test/raptor.yml
@ -752,6 +752,244 @@ raptor-tp6m-10-fennec:
            - --app=fennec
            - --binary=org.mozilla.fennec_aurora

+raptor-tp6m-1-fenix-cold:
+    description: "Raptor tp6m-1 cold page-load on Fenix"
+    try-name: raptor-tp6m-1-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-1)
+    run-on-projects: ['try']
+    e10s: true
+    target:
+        index: project.mobile.fenix.branch.master.latest.greenfield.arm-release-raptor
+        name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-1
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix
+            - --activity=browser.BrowserPerformanceTestActivity
+
+raptor-tp6m-2-fenix-cold:
+    description: "Raptor tp6m-2 cold page-load on Fenix"
+    try-name: raptor-tp6m-2-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-2)
+    run-on-projects: ['try']
+    e10s: true
+    target:
+        index: project.mobile.fenix.branch.master.latest.greenfield.arm-release-raptor
+        name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-2
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix
+            - --activity=browser.BrowserPerformanceTestActivity
+
+raptor-tp6m-3-fenix-cold:
+    description: "Raptor tp6m-3 cold page-load on Fenix"
+    try-name: raptor-tp6m-3-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-3)
+    run-on-projects: ['try']
+    e10s: true
+    target:
+        index: project.mobile.fenix.branch.master.latest.greenfield.arm-release-raptor
+        name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-3
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix
+            - --activity=browser.BrowserPerformanceTestActivity
+
+raptor-tp6m-4-fenix-cold:
+    description: "Raptor tp6m-4 cold page-load on Fenix"
+    try-name: raptor-tp6m-4-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-4)
+    run-on-projects: ['try']
+    e10s: true
+    target:
+        index: project.mobile.fenix.branch.master.latest.greenfield.arm-release-raptor
+        name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-4
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix
+            - --activity=browser.BrowserPerformanceTestActivity
+
+raptor-tp6m-5-fenix-cold:
+    description: "Raptor tp6m-5 cold page-load on Fenix"
+    try-name: raptor-tp6m-5-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-5)
+    run-on-projects: ['try']
+    e10s: true
+    target:
+        index: project.mobile.fenix.branch.master.latest.greenfield.arm-release-raptor
+        name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-5
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix
+            - --activity=browser.BrowserPerformanceTestActivity
+
+raptor-tp6m-6-fenix-cold:
+    description: "Raptor tp6m-6 cold page-load on Fenix"
+    try-name: raptor-tp6m-6-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-6)
+    run-on-projects: ['try']
+    e10s: true
+    target:
+        index: project.mobile.fenix.branch.master.latest.greenfield.arm-release-raptor
+        name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-6
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix
+            - --activity=browser.BrowserPerformanceTestActivity
+
+raptor-tp6m-7-fenix-cold:
+    description: "Raptor tp6m-7 cold page-load on Fenix"
+    try-name: raptor-tp6m-7-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-7)
+    run-on-projects: ['try']
+    e10s: true
+    target:
+        index: project.mobile.fenix.branch.master.latest.greenfield.arm-release-raptor
+        name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-7
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix
+            - --activity=browser.BrowserPerformanceTestActivity
+
+raptor-tp6m-8-fenix-cold:
+    description: "Raptor tp6m-8 cold page-load on Fenix"
+    try-name: raptor-tp6m-8-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-8)
+    run-on-projects: ['try']
+    e10s: true
+    target:
+        index: project.mobile.fenix.branch.master.latest.greenfield.arm-release-raptor
+        name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-8
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix
+            - --activity=browser.BrowserPerformanceTestActivity
+
+raptor-tp6m-9-fenix-cold:
+    description: "Raptor tp6m-9 cold page-load on Fenix"
+    try-name: raptor-tp6m-9-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-9)
+    run-on-projects: ['try']
+    e10s: true
+    target:
+        index: project.mobile.fenix.branch.master.latest.greenfield.arm-release-raptor
+        name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-9
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix
+            - --activity=browser.BrowserPerformanceTestActivity
+
+raptor-tp6m-10-fenix-cold:
+    description: "Raptor tp6m-10 cold page-load on Fenix"
+    try-name: raptor-tp6m-10-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-10)
+    run-on-projects: ['try']
+    e10s: true
+    target:
+        index: project.mobile.fenix.branch.master.latest.greenfield.arm-release-raptor
+        name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-10
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix
+            - --activity=browser.BrowserPerformanceTestActivity
+
+raptor-tp6m-11-fenix-cold:
+    description: "Raptor tp6m-11 cold page-load on Fenix"
+    try-name: raptor-tp6m-11-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-11)
+    run-on-projects: ['try']
+    e10s: true
+    target:
+        index: project.mobile.fenix.branch.master.latest.greenfield.arm-release-raptor
+        name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-11
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix
+            - --activity=browser.BrowserPerformanceTestActivity
+
+raptor-tp6m-12-fenix-cold:
+    description: "Raptor tp6m-12 cold page-load on Fenix"
+    try-name: raptor-tp6m-12-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-12)
+    run-on-projects: ['try']
+    e10s: true
+    target:
+        index: project.mobile.fenix.branch.master.latest.greenfield.arm-release-raptor
+        name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-12
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix
+            - --activity=browser.BrowserPerformanceTestActivity
+
+raptor-tp6m-13-fenix-cold:
+    description: "Raptor tp6m-13 cold page-load on Fenix"
+    try-name: raptor-tp6m-13-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-13)
+    run-on-projects: ['try']
+    e10s: true
+    target:
+        index: project.mobile.fenix.branch.master.latest.greenfield.arm-release-raptor
+        name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-13
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix
+            - --activity=browser.BrowserPerformanceTestActivity
+
+raptor-tp6m-14-fenix-cold:
+    description: "Raptor tp6m-14 cold page-load on Fenix"
+    try-name: raptor-tp6m-14-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-14)
+    run-on-projects: ['try']
+    e10s: true
+    target:
+        index: project.mobile.fenix.branch.master.latest.greenfield.arm-release-raptor
+        name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-14
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix
+            - --activity=browser.BrowserPerformanceTestActivity
+
 raptor-speedometer-firefox:
    description: "Raptor Speedometer on Firefox"
    try-name: raptor-speedometer-firefox
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@ -470,6 +470,20 @@ android-hw-arm7-raptor:
    - raptor-tp6m-8-refbrow
    - raptor-tp6m-9-refbrow
    - raptor-tp6m-10-refbrow
+    - raptor-tp6m-1-fenix-cold
+    - raptor-tp6m-2-fenix-cold
+    - raptor-tp6m-3-fenix-cold
+    - raptor-tp6m-4-fenix-cold
+    - raptor-tp6m-5-fenix-cold
+    - raptor-tp6m-6-fenix-cold
+    - raptor-tp6m-7-fenix-cold
+    - raptor-tp6m-8-fenix-cold
+    - raptor-tp6m-9-fenix-cold
+    - raptor-tp6m-10-fenix-cold
+    - raptor-tp6m-11-fenix-cold
+    - raptor-tp6m-12-fenix-cold
+    - raptor-tp6m-13-fenix-cold
+    - raptor-tp6m-14-fenix-cold

 android-hw-aarch64-raptor:
    - raptor-speedometer-geckoview
@ -495,6 +509,20 @@ android-hw-aarch64-raptor:
    - raptor-tp6m-8-refbrow
    - raptor-tp6m-9-refbrow
    - raptor-tp6m-10-refbrow
+    - raptor-tp6m-1-fenix-cold
+    - raptor-tp6m-2-fenix-cold
+    - raptor-tp6m-3-fenix-cold
+    - raptor-tp6m-4-fenix-cold
+    - raptor-tp6m-5-fenix-cold
+    - raptor-tp6m-6-fenix-cold
+    - raptor-tp6m-7-fenix-cold
+    - raptor-tp6m-8-fenix-cold
+    - raptor-tp6m-9-fenix-cold
+    - raptor-tp6m-10-fenix-cold
+    - raptor-tp6m-11-fenix-cold
+    - raptor-tp6m-12-fenix-cold
+    - raptor-tp6m-13-fenix-cold
+    - raptor-tp6m-14-fenix-cold

 android-hw-arm7-raptor-power:
    - raptor-speedometer-geckoview-power
--- a/testing/raptor/raptor/raptor.ini
+++ b/testing/raptor/raptor/raptor.ini
@ -28,6 +28,19 @@

 # raptor cold pageload tests mobile
 [include:tests/raptor-tp6m-cold-1.ini]
+[include:tests/raptor-tp6m-cold-2.ini]
+[include:tests/raptor-tp6m-cold-3.ini]
+[include:tests/raptor-tp6m-cold-4.ini]
+[include:tests/raptor-tp6m-cold-5.ini]
+[include:tests/raptor-tp6m-cold-6.ini]
+[include:tests/raptor-tp6m-cold-7.ini]
+[include:tests/raptor-tp6m-cold-8.ini]
+[include:tests/raptor-tp6m-cold-9.ini]
+[include:tests/raptor-tp6m-cold-10.ini]
+[include:tests/raptor-tp6m-cold-11.ini]
+[include:tests/raptor-tp6m-cold-12.ini]
+[include:tests/raptor-tp6m-cold-13.ini]
+[include:tests/raptor-tp6m-cold-14.ini]

 # raptor benchmark tests
 [include:tests/raptor-assorted-dom.ini]
--- a/testing/raptor/raptor/tests/raptor-tp6m-cold-1.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6m-cold-1.ini
@ -8,7 +8,7 @@
 type =  pageload
 playback = mitmproxy-android
 playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
-browser_cycles = 10
+browser_cycles = 15
 unit = ms
 lower_is_better = true
 alert_threshold = 2.0
@ -30,9 +30,16 @@ playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-facebook.manifest
 playback_recordings = android-facebook.mp
 measure = fnbpaint, fcp, dcf, loadtime

-[raptor-tp6m-cold-google-geckoview]
-apps = geckoview
-test_url = https://www.google.com
-playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-google.manifest
-playback_recordings = android-google.mp
+[raptor-tp6m-cold-amazon-fenix]
+apps = fenix
+test_url = https://www.amazon.com
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-amazon.manifest
+playback_recordings = android-amazon.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-cold-facebook-fenix]
+apps = fenix
+test_url = https://m.facebook.com
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-facebook.manifest
+playback_recordings = android-facebook.mp
 measure = fnbpaint, fcp, dcf, loadtime
--- a/testing/raptor/raptor/tests/raptor-tp6m-cold-10.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6m-cold-10.ini
@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-cold-10
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+browser_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6m-bbc-fenix]
+apps = fenix
+test_url = https://www.bbc.com/news/business-47245877
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-bbc.manifest
+playback_recordings = mitmproxy-recordings-raptor-tp6m-bbc.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-stackoverflow-fenix]
+apps = fenix
+test_url = https://stackoverflow.com/
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-stackoverflow.manifest
+playback_recordings = mitmproxy-recordings-raptor-tp6m-stackoverflow.mp
+measure = fnbpaint, fcp, dcf, loadtime
--- a/testing/raptor/raptor/tests/raptor-tp6m-cold-11.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6m-cold-11.ini
@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-cold-11
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+browser_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6m-microsoft-support-fenix]
+apps = fenix
+test_url = https://support.microsoft.com/en-us
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-microsoft-support.manifest
+playback_recordings = android-microsoft-support.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-jianshu-fenix]
+apps = fenix
+test_url = https://www.jianshu.com/
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-jianshu.manifest
+playback_recordings = android-jianshu.mp
+measure = fnbpaint, fcp, dcf, loadtime
--- a/testing/raptor/raptor/tests/raptor-tp6m-cold-12.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6m-cold-12.ini
@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-cold-12
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+browser_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6m-imdb-fenix]
+apps = fenix
+test_url = https://m.imdb.com/
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-imdb.manifest
+playback_recordings = android-imdb.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-allrecipes-fenix]
+apps = fenix
+test_url = https://www.allrecipes.com/
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-allrecipes.manifest
+playback_recordings = android-allrecipes.mp
+measure = fnbpaint, fcp, dcf, loadtime
--- a/testing/raptor/raptor/tests/raptor-tp6m-cold-13.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6m-cold-13.ini
@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-cold-13
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+browser_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6m-espn-fenix]
+apps = fenix
+test_url = http://www.espn.com/nba/story/_/page/allstarweekend25788027/the-comparison-lebron-james-michael-jordan-their-own-words
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-espn.manifest
+playback_recordings = android-espn.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-web-de-fenix]
+apps = fenix
+test_url = https://web.de/magazine/politik/politologe-glaubt-grossen-koalition-herbst-knallen-33563566
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-web-de.manifest
+playback_recordings = android-web-de.mp
+measure = fnbpaint, fcp, dcf, loadtime
--- a/testing/raptor/raptor/tests/raptor-tp6m-cold-14.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6m-cold-14.ini
@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-cold-14
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+browser_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6m-facebook-cristiano-fenix]
+apps = fenix
+test_url = https://m.facebook.com/Cristiano
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-facebook-cristiano.manifest
+playback_recordings = android-facebook-cristiano.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-aframeio-animation-fenix]
+apps = fenix
+test_url = https://aframe.io/examples/showcase/animation
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-aframeio-animation.manifest
+playback_recordings = android-aframeio-animation.mp
+measure = fnbpaint, fcp, dcf, loadtime
--- a/testing/raptor/raptor/tests/raptor-tp6m-cold-2.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6m-cold-2.ini
@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-cold-2
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+browser_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6m-google-fenix]
+apps = fenix
+test_url = https://www.google.com
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-google.manifest
+playback_recordings = android-google.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-youtube-fenix]
+apps = fenix
+test_url = https://www.youtube.com
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-youtube.manifest
+playback_recordings = android-youtube.mp
+measure = fnbpaint, fcp, dcf, loadtime
--- a/testing/raptor/raptor/tests/raptor-tp6m-cold-3.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6m-cold-3.ini
@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-cold-3
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+browser_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6m-instagram-fenix]
+apps = fenix
+test_url = https://www.instagram.com
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-instagram-mobile.manifest
+playback_recordings = instagram-mobile.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-bing-fenix]
+apps = fenix
+test_url = https://www.bing.com
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-bing-mobile.manifest
+playback_recordings = bing-mobile.mp
+measure = fnbpaint, fcp, dcf, loadtime
--- a/testing/raptor/raptor/tests/raptor-tp6m-cold-4.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6m-cold-4.ini
@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-cold-4
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+browser_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6m-bing-restaurants-fenix]
+apps = fenix
+test_url = https://www.bing.com/search?q=restaurants
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-bing-restaurants-mobile.manifest
+playback_recordings = bing-restaurants-mobile.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-ebay-kleinanzeigen-fenix]
+apps = fenix
+test_url = https://m.ebay-kleinanzeigen.de
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-ebay-kleinanzeigen-mobile.manifest
+playback_recordings = ebay-kleinanzeigen-mobile.mp
+measure = fnbpaint, fcp, dcf, loadtime
--- a/testing/raptor/raptor/tests/raptor-tp6m-cold-5.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6m-cold-5.ini
@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-cold-5
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+browser_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6m-ebay-kleinanzeigen-search-fenix]
+apps = fenix
+test_url = https://m.ebay-kleinanzeigen.de/s-anzeigen/auf-zeit-wg-berlin/zimmer/c199-l3331
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-ebay-kleinanzeigen-search-mobile.manifest
+playback_recordings = ebay-kleinanzeigen-search-mobile.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-google-maps-fenix]
+apps = fenix
+test_url = https://www.google.com/maps?force=pwa
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-google-maps.manifest
+playback_recordings = google_maps_mobile.mp
+measure = fnbpaint, fcp, dcf, loadtime
--- a/testing/raptor/raptor/tests/raptor-tp6m-cold-6.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6m-cold-6.ini
@ -0,0 +1,32 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-cold-6
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+browser_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6m-google-restaurants-fenix]
+apps = fenix
+test_url = https://www.google.com/search?q=restaurants+near+me
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-google-restaurants.manifest
+playback_recordings = google-search-restaurants-mobile.mp
+measure = fnbpaint, fcp, dcf, loadtime
+disabled = Bug 1533283 Intermittent timeouts running raptor-tp6m-google-restaurants-geckoview
+
+[raptor-tp6m-amazon-search-fenix]
+apps = fenix
+test_url = https://www.amazon.com/s/ref=nb_sb_noss_2/139-6317191-5622045?url=search-alias%3Daps&field-keywords=mobile+phone
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-amazon-search.manifest
+playback_recordings = android-amazon-search.mp
+measure = fnbpaint, fcp, dcf, loadtime
--- a/testing/raptor/raptor/tests/raptor-tp6m-cold-7.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6m-cold-7.ini
@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-cold-7
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+browser_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6m-wikipedia-fenix]
+apps = fenix
+test_url = https://en.m.wikipedia.org/wiki/Main_Page
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-wikipedia.manifest
+playback_recordings = android-wikipedia.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-youtube-watch-fenix]
+apps = fenix
+test_url = https://www.youtube.com/watch?v=COU5T-Wafa4
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-youtube-watch.manifest
+playback_recordings = android-youtube-watch.mp
+measure = fnbpaint, fcp, dcf, loadtime
--- a/testing/raptor/raptor/tests/raptor-tp6m-cold-8.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6m-cold-8.ini
@ -0,0 +1,32 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-cold-8
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+browser_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6m-booking-fenix]
+apps = fenix
+test_url = https://www.booking.com/
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-booking.manifest
+playback_recordings = android-booking.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-cnn-fenix]
+apps = fenix
+test_url = https://edition.cnn.com/
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-cnn.manifest
+playback_recordings = android-cnn.mp
+measure = fnbpaint, fcp, dcf, loadtime
+disabled = Bug 1533287 Intermittent timeouts running raptor-tp6m-cnn-geckoview
--- a/testing/raptor/raptor/tests/raptor-tp6m-cold-9.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6m-cold-9.ini
@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-cold-9
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+browser_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6m-cnn-ampstories-fenix]
+apps = fenix
+test_url = https://edition.cnn.com/ampstories/us/why-hurricane-michael-is-a-monster-unlike-any-other
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-cnn-ampstories.manifest
+playback_recordings = android-cnn-ampstories.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-reddit-fenix]
+apps = fenix
+test_url = https://www.reddit.com
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-reddit.manifest
+playback_recordings = mitmproxy-recordings-raptor-tp6m-reddit.mp
+measure = fnbpaint, fcp, dcf, loadtime