Bug 1436827 - add new raptor 'gdocs' pageload test suite (for google docs/slides/sheets); r=igoldan

MozReview-Commit-ID: EfhBJPgpyRc

--HG--
extra : rebase_source : 73e37a56af23f1a287e84fba4de4bf35016acba9

taskcluster/ci/test/raptor.yml

@@ -25,9 +25,9 @@ job-defaults:
                 default:
                     - raptor/linux_config.py
 
-raptor-firefox-tp6:
-    description: "Raptor Firefox tp6"
-    try-name: raptor-firefox-tp6
+raptor-tp6-firefox:
+    description: "Raptor tp6 on Firefox"
+    try-name: raptor-tp6-firefox
     treeherder-symbol: Rap(tp6)
     run-on-projects: ['try', 'mozilla-central']
     max-run-time: 1200
@@ -35,9 +35,20 @@ raptor-firefox-tp6:
         extra-options:
             - --test=raptor-tp6
 
-raptor-firefox-speedometer:
-    description: "Raptor Firefox Speedometer"
-    try-name: raptor-firefox-speedometer
+raptor-tp6-chrome:
+    description: "Raptor tp6 on Chrome"
+    try-name: raptor-tp6-chrome
+    treeherder-symbol: Rap-C(tp6)
+    run-on-projects: ['try']
+    max-run-time: 1200
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6
+            - --app=chrome
+
+raptor-speedometer-firefox:
+    description: "Raptor Speedometer on Firefox"
+    try-name: raptor-speedometer-firefox
     treeherder-symbol: Rap(sp)
     run-on-projects: ['try', 'mozilla-central']
     max-run-time: 1500
@@ -45,73 +56,9 @@ raptor-firefox-speedometer:
         extra-options:
             - --test=raptor-speedometer
 
-raptor-firefox-stylebench:
-    description: "Raptor Firefox StyleBench"
-    try-name: raptor-firefox-stylebench
-    treeherder-symbol: Rap(sb)
-    run-on-projects: ['try', 'mozilla-central']
-    max-run-time: 1800
-    mozharness:
-        extra-options:
-            - --test=raptor-stylebench
-
-raptor-firefox-sunspider:
-    description: "Raptor Firefox SunSpider"
-    try-name: raptor-firefox-sunspider
-    treeherder-symbol: Rap(ss)
-    run-on-projects: ['try', 'mozilla-central']
-    max-run-time: 1800
-    mozharness:
-        extra-options:
-            - --test=raptor-sunspider
-
-raptor-firefox-motionmark-htmlsuite:
-    description: "Raptor Firefox MotionMark HtmlSuite"
-    try-name: raptor-firefox-motionmark-htmlsuite
-    treeherder-symbol: Rap(mm-h)
-    run-on-projects: ['try', 'mozilla-central']
-    max-run-time: 1800
-    mozharness:
-        extra-options:
-            - --test=raptor-motionmark-htmlsuite
-
-raptor-firefox-motionmark-animometer:
-    description: "Raptor Firefox MotionMark Animometer"
-    try-name: raptor-firefox-motionmark-animometer
-    treeherder-symbol: Rap(mm-a)
-    run-on-projects: ['try', 'mozilla-central']
-    max-run-time: 1800
-    mozharness:
-        extra-options:
-            - --test=raptor-motionmark-animometer
-
-raptor-firefox-webaudio:
-    description: "Raptor Firefox WebAudio benchmark"
-    try-name: raptor-firefox-webaudio
-    treeherder-symbol: Rap(wa)
-    run-on-projects: ['try', 'mozilla-central']
-    max-run-time: 1800
-    mozharness:
-        extra-options:
-            - --test=raptor-webaudio
-
-raptor-chrome-tp6:
-    description: "Raptor Chrome tp6"
-    try-name: raptor-chrome-tp6
-    treeherder-symbol: Rap-C(tp6)
-    run-on-projects:
-        by-test-platform:
-            .*-qr/.*: ['try']
-            default: ['try']
-    max-run-time: 1200
-    mozharness:
-        extra-options:
-            - --test=raptor-tp6
-            - --app=chrome
-
-raptor-chrome-speedometer:
-    description: "Raptor Chrome Speedometer"
-    try-name: raptor-chrome-speedometer
+raptor-speedometer-chrome:
+    description: "Raptor Speedometer on Chrome"
+    try-name: raptor-speedometer-chrome
     treeherder-symbol: Rap-C(sp)
     run-on-projects: ['try', 'mozilla-central']
     max-run-time: 1500
@@ -120,9 +67,19 @@ raptor-chrome-speedometer:
             - --test=raptor-speedometer
             - --app=chrome
 
-raptor-chrome-stylebench:
-    description: "Raptor Chrome StyleBench"
-    try-name: raptor-chrome-stylebench
+raptor-stylebench-firefox:
+    description: "Raptor StyleBench on Firefox"
+    try-name: raptor-stylebench-firefox
+    treeherder-symbol: Rap(sb)
+    run-on-projects: ['try', 'mozilla-central']
+    max-run-time: 1800
+    mozharness:
+        extra-options:
+            - --test=raptor-stylebench
+
+raptor-stylebench-chrome:
+    description: "Raptor StyleBench on Chrome"
+    try-name: raptor-stylebench-chrome
     treeherder-symbol: Rap-C(sb)
     run-on-projects: ['try', 'mozilla-central']
     max-run-time: 1800
@@ -131,20 +88,19 @@ raptor-chrome-stylebench:
             - --test=raptor-stylebench
             - --app=chrome
 
-raptor-chrome-sunspider:
-    description: "Raptor Chrome SunSpider"
-    try-name: raptor-chrome-sunspider
-    treeherder-symbol: Rap-C(ss)
+raptor-motionmark-htmlsuite-firefox:
+    description: "Raptor MotionMark HtmlSuite on Firefox"
+    try-name: raptor-motionmark-htmlsuite-firefox
+    treeherder-symbol: Rap(mm-h)
     run-on-projects: ['try', 'mozilla-central']
     max-run-time: 1800
     mozharness:
         extra-options:
-            - --test=raptor-sunspider
-            - --app=chrome
+            - --test=raptor-motionmark-htmlsuite
 
-raptor-chrome-motionmark-htmlsuite:
-    description: "Raptor Chrome MotionMark HtmlSuite"
-    try-name: raptor-chrome-motionmark-htmlsuite
+raptor-motionmark-htmlsuite-chrome:
+    description: "Raptor MotionMark HtmlSuite on Chrome"
+    try-name: raptor-motionmark-htmlsuite-chrome
     treeherder-symbol: Rap-C(mm-h)
     run-on-projects: ['try', 'mozilla-central']
     max-run-time: 1800
@@ -153,9 +109,19 @@ raptor-chrome-motionmark-htmlsuite:
             - --test=raptor-motionmark-htmlsuite
             - --app=chrome
 
-raptor-chrome-motionmark-animometer:
-    description: "Raptor Chrome MotionMark Animometer"
-    try-name: raptor-chrome-motionmark-animometer
+raptor-motionmark-animometer-firefox:
+    description: "Raptor MotionMark Animometer on Firefox"
+    try-name: raptor-motionmark-animometer-firefox
+    treeherder-symbol: Rap(mm-a)
+    run-on-projects: ['try', 'mozilla-central']
+    max-run-time: 1800
+    mozharness:
+        extra-options:
+            - --test=raptor-motionmark-animometer
+
+raptor-motionmark-animometer-chrome:
+    description: "Raptor MotionMark Animometer on Chrome"
+    try-name: raptor-motionmark-animometer-chrome
     treeherder-symbol: Rap-C(mm-a)
     run-on-projects: ['try', 'mozilla-central']
     max-run-time: 1800
@@ -164,9 +130,19 @@ raptor-chrome-motionmark-animometer:
             - --test=raptor-motionmark-animometer
             - --app=chrome
 
-raptor-chrome-webaudio:
-    description: "Raptor Chrome WebAudio benchmark"
-    try-name: raptor-chrome-webaudio
+raptor-webaudio-firefox:
+    description: "Raptor WebAudio on Firefox"
+    try-name: raptor-webaudio-firefox
+    treeherder-symbol: Rap(wa)
+    run-on-projects: ['try', 'mozilla-central']
+    max-run-time: 1800
+    mozharness:
+        extra-options:
+            - --test=raptor-webaudio
+
+raptor-webaudio-chrome:
+    description: "Raptor WebAudio on Chrome"
+    try-name: raptor-webaudio-chrome
     treeherder-symbol: Rap-C(wa)
     run-on-projects: ['try', 'mozilla-central']
     max-run-time: 1800
@@ -174,3 +150,45 @@ raptor-chrome-webaudio:
         extra-options:
             - --test=raptor-webaudio
             - --app=chrome
+
+raptor-gdocs-firefox:
+    description: "Raptor gdocs on Firefox"
+    try-name: raptor-gdocs-firefox
+    treeherder-symbol: Rap(gdocs)
+    run-on-projects: ['try', 'mozilla-central']
+    max-run-time: 1800
+    mozharness:
+        extra-options:
+            - --test=raptor-gdocs
+
+raptor-gdocs-chrome:
+    description: "Raptor gdocs on Chrome"
+    try-name: raptor-gdocs-chrome
+    treeherder-symbol: Rap-C(gdocs)
+    run-on-projects: ['try']
+    max-run-time: 1800
+    mozharness:
+        extra-options:
+            - --test=raptor-gdocs
+            - --app=chrome
+
+raptor-sunspider-firefox:
+    description: "Raptor SunSpider on Firefox"
+    try-name: raptor-sunspider-firefox
+    treeherder-symbol: Rap(ss)
+    run-on-projects: ['try', 'mozilla-central']
+    max-run-time: 1800
+    mozharness:
+        extra-options:
+            - --test=raptor-sunspider
+
+raptor-sunspider-chrome:
+    description: "Raptor SunSpider on Chrome"
+    try-name: raptor-sunspider-chrome
+    treeherder-symbol: Rap-C(ss)
+    run-on-projects: ['try', 'mozilla-central']
+    max-run-time: 1800
+    mozharness:
+        extra-options:
+            - --test=raptor-sunspider
+            - --app=chrome

taskcluster/ci/test/test-sets.yml

@@ -81,18 +81,22 @@ talos:
     - talos-h2
 
 raptor:
-    - raptor-firefox-tp6
-    - raptor-firefox-speedometer
-    - raptor-firefox-stylebench
-    - raptor-firefox-sunspider
-    - raptor-firefox-motionmark-htmlsuite
-    - raptor-firefox-motionmark-animometer
-    - raptor-firefox-webaudio
-    - raptor-chrome-tp6
-    - raptor-chrome-speedometer
-    - raptor-chrome-stylebench
-    - raptor-chrome-sunspider
-    - raptor-chrome-webaudio
+    - raptor-tp6-firefox
+    - raptor-tp6-chrome
+    - raptor-speedometer-firefox
+    - raptor-speedometer-chrome
+    - raptor-stylebench-firefox
+    - raptor-stylebench-chrome
+    - raptor-motionmark-htmlsuite-firefox
+    - raptor-motionmark-htmlsuite-chrome
+    - raptor-motionmark-animometer-firefox
+    - raptor-motionmark-animometer-chrome
+    - raptor-webaudio-firefox
+    - raptor-webaudio-chrome
+    - raptor-gdocs-firefox
+    - raptor-gdocs-chrome
+    - raptor-sunspider-firefox
+    - raptor-sunspider-chrome
 
 awsy:
     - awsy

testing/raptor/raptor/playback/mitmproxy-recordings-raptor-gdocs.manifest

@@ -0,0 +1,9 @@
+[
+    {
+        "filename": "mitmproxy-recordings-raptor-gdocs.zip",
+        "size": 26179496,
+        "digest": "eb32218815d3c2187807f3a18d6df1657e01a215820deaaa7a035f06d180780ac9652479ac7db1d8cde3728a890521aff2e0a5c4738304966b2a93ce1b7f33bb",
+        "algorithm": "sha512",
+        "unpack": true
+    }
+]

testing/raptor/raptor/raptor.ini

@@ -6,3 +6,4 @@
 [include:tests/raptor-motionmark-htmlsuite.ini]
 [include:tests/raptor-motionmark-animometer.ini]
 [include:tests/raptor-webaudio.ini]
+[include:tests/raptor-gdocs.ini]

testing/raptor/raptor/tests/raptor-gdocs.ini

@@ -0,0 +1,53 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor gdocs
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+python3_win_manifest = python3{x64}.manifest
+playback_pageset_manifest = mitmproxy-recordings-raptor-gdocs.manifest
+page_cycles = 25
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 30000
+
+[raptor-google-docs-firefox]
+apps = firefox
+test_url = https://docs.google.com/document/d/1US-07msg12slQtI_xchzYxcKlTs6Fp7WqIc6W5GK5M8/edit?usp=sharing
+playback_recordings = google-docs.mp
+measure = fnbpaint
+
+[raptor-google-sheets-firefox]
+apps = firefox
+test_url = https://docs.google.com/spreadsheets/d/1jT9qfZFAeqNoOK97gruc34Zb7y_Q-O_drZ8kSXT-4D4/edit?usp=sharing
+playback_recordings = google-sheets.mp
+measure = fnbpaint
+
+[raptor-google-slides-firefox]
+apps = firefox
+test_url = https://docs.google.com/presentation/d/1Ici0ceWwpFvmIb3EmKeWSq_vAQdmmdFcWqaiLqUkJng/edit?usp=sharing
+playback_recordings = google-slides.mp
+measure = fnbpaint
+
+[raptor-google-docs-chrome]
+apps = chrome
+test_url = https://docs.google.com/document/d/1US-07msg12slQtI_xchzYxcKlTs6Fp7WqIc6W5GK5M8/edit?usp=sharing
+playback_recordings = google-docs.mp
+measure = fcp
+
+[raptor-google-sheets-chrome]
+apps = chrome
+test_url = https://docs.google.com/spreadsheets/d/1jT9qfZFAeqNoOK97gruc34Zb7y_Q-O_drZ8kSXT-4D4/edit?usp=sharing
+playback_recordings = google-sheets.mp
+measure = fcp
+
+[raptor-google-slides-chrome]
+apps = chrome
+test_url = https://docs.google.com/presentation/d/1Ici0ceWwpFvmIb3EmKeWSq_vAQdmmdFcWqaiLqUkJng/edit?usp=sharing
+playback_recordings = google-slides.mp
+measure = fcp

testing/raptor/raptor/tests/raptor-tp6.ini

@@ -15,19 +15,19 @@ unit = ms
 lower_is_better = true
 alert_threshold = 2.0
 
-[raptor-firefox-tp6-amazon]
+[raptor-tp6-amazon-firefox]
 apps = firefox
 test_url = https://www.amazon.com/s/url=search-alias%3Daps&field-keywords=laptop
 playback_recordings = amazon.mp
 measure = fnbpaint
 
-[raptor-firefox-tp6-facebook]
+[raptor-tp6-facebook-firefox]
 apps = firefox
 test_url = https://www.facebook.com
 playback_recordings = facebook.mp
 measure = fnbpaint
 
-[raptor-firefox-tp6-google]
+[raptor-tp6-google-firefox]
 apps = firefox
 # note: use the full url as the first part (without '&cad=h') redirects
 # to the url + '&cad=h'; that redirection causes measure.js content
@@ -36,31 +36,31 @@ test_url = https://www.google.com/search?hl=en&q=barack+obama&cad=h
 playback_recordings = google-search.mp
 measure = fnbpaint
 
-[raptor-firefox-tp6-youtube]
+[raptor-tp6-youtube-firefox]
 apps = firefox
 test_url = https://www.youtube.com
 playback_recordings = youtube.mp
 measure = fnbpaint
 
-[raptor-chrome-tp6-amazon]
+[raptor-tp6-amazon-chrome]
 apps = chrome
 test_url = https://www.amazon.com/s/url=search-alias%3Daps&field-keywords=laptop
 playback_recordings = amazon.mp
 measure = fcp
 
-[raptor-chrome-tp6-facebook]
+[raptor-tp6-facebook-chrome]
 apps = chrome
 test_url = https://www.facebook.com
 playback_recordings = facebook.mp
 measure = fcp
 
-[raptor-chrome-tp6-google]
+[raptor-tp6-google-chrome]
 apps = chrome
 test_url = https://www.google.com/#hl=en&q=barack+obama
 playback_recordings = google-search.mp
 measure = fcp
 
-[raptor-chrome-tp6-youtube]
+[raptor-tp6-youtube-chrome]
 apps = chrome
 test_url = https://www.youtube.com
 playback_recordings = youtube.mp