diff --git a/taskcluster/ci/test/raptor.yml b/taskcluster/ci/test/raptor.yml index aaedc542d0ba..d958a326d792 100644 --- a/taskcluster/ci/test/raptor.yml +++ b/taskcluster/ci/test/raptor.yml @@ -25,9 +25,9 @@ job-defaults: default: - raptor/linux_config.py -raptor-firefox-tp6: - description: "Raptor Firefox tp6" - try-name: raptor-firefox-tp6 +raptor-tp6-firefox: + description: "Raptor tp6 on Firefox" + try-name: raptor-tp6-firefox treeherder-symbol: Rap(tp6) run-on-projects: ['try', 'mozilla-central'] max-run-time: 1200 @@ -35,9 +35,20 @@ raptor-firefox-tp6: extra-options: - --test=raptor-tp6 -raptor-firefox-speedometer: - description: "Raptor Firefox Speedometer" - try-name: raptor-firefox-speedometer +raptor-tp6-chrome: + description: "Raptor tp6 on Chrome" + try-name: raptor-tp6-chrome + treeherder-symbol: Rap-C(tp6) + run-on-projects: ['try'] + max-run-time: 1200 + mozharness: + extra-options: + - --test=raptor-tp6 + - --app=chrome + +raptor-speedometer-firefox: + description: "Raptor Speedometer on Firefox" + try-name: raptor-speedometer-firefox treeherder-symbol: Rap(sp) run-on-projects: ['try', 'mozilla-central'] max-run-time: 1500 @@ -45,73 +56,9 @@ raptor-firefox-speedometer: extra-options: - --test=raptor-speedometer -raptor-firefox-stylebench: - description: "Raptor Firefox StyleBench" - try-name: raptor-firefox-stylebench - treeherder-symbol: Rap(sb) - run-on-projects: ['try', 'mozilla-central'] - max-run-time: 1800 - mozharness: - extra-options: - - --test=raptor-stylebench - -raptor-firefox-sunspider: - description: "Raptor Firefox SunSpider" - try-name: raptor-firefox-sunspider - treeherder-symbol: Rap(ss) - run-on-projects: ['try', 'mozilla-central'] - max-run-time: 1800 - mozharness: - extra-options: - - --test=raptor-sunspider - -raptor-firefox-motionmark-htmlsuite: - description: "Raptor Firefox MotionMark HtmlSuite" - try-name: raptor-firefox-motionmark-htmlsuite - treeherder-symbol: Rap(mm-h) - run-on-projects: ['try', 'mozilla-central'] - max-run-time: 1800 - mozharness: - extra-options: - - --test=raptor-motionmark-htmlsuite - -raptor-firefox-motionmark-animometer: - description: "Raptor Firefox MotionMark Animometer" - try-name: raptor-firefox-motionmark-animometer - treeherder-symbol: Rap(mm-a) - run-on-projects: ['try', 'mozilla-central'] - max-run-time: 1800 - mozharness: - extra-options: - - --test=raptor-motionmark-animometer - -raptor-firefox-webaudio: - description: "Raptor Firefox WebAudio benchmark" - try-name: raptor-firefox-webaudio - treeherder-symbol: Rap(wa) - run-on-projects: ['try', 'mozilla-central'] - max-run-time: 1800 - mozharness: - extra-options: - - --test=raptor-webaudio - -raptor-chrome-tp6: - description: "Raptor Chrome tp6" - try-name: raptor-chrome-tp6 - treeherder-symbol: Rap-C(tp6) - run-on-projects: - by-test-platform: - .*-qr/.*: ['try'] - default: ['try'] - max-run-time: 1200 - mozharness: - extra-options: - - --test=raptor-tp6 - - --app=chrome - -raptor-chrome-speedometer: - description: "Raptor Chrome Speedometer" - try-name: raptor-chrome-speedometer +raptor-speedometer-chrome: + description: "Raptor Speedometer on Chrome" + try-name: raptor-speedometer-chrome treeherder-symbol: Rap-C(sp) run-on-projects: ['try', 'mozilla-central'] max-run-time: 1500 @@ -120,9 +67,19 @@ raptor-chrome-speedometer: - --test=raptor-speedometer - --app=chrome -raptor-chrome-stylebench: - description: "Raptor Chrome StyleBench" - try-name: raptor-chrome-stylebench +raptor-stylebench-firefox: + description: "Raptor StyleBench on Firefox" + try-name: raptor-stylebench-firefox + treeherder-symbol: Rap(sb) + run-on-projects: ['try', 'mozilla-central'] + max-run-time: 1800 + mozharness: + extra-options: + - --test=raptor-stylebench + +raptor-stylebench-chrome: + description: "Raptor StyleBench on Chrome" + try-name: raptor-stylebench-chrome treeherder-symbol: Rap-C(sb) run-on-projects: ['try', 'mozilla-central'] max-run-time: 1800 @@ -131,20 +88,19 @@ raptor-chrome-stylebench: - --test=raptor-stylebench - --app=chrome -raptor-chrome-sunspider: - description: "Raptor Chrome SunSpider" - try-name: raptor-chrome-sunspider - treeherder-symbol: Rap-C(ss) +raptor-motionmark-htmlsuite-firefox: + description: "Raptor MotionMark HtmlSuite on Firefox" + try-name: raptor-motionmark-htmlsuite-firefox + treeherder-symbol: Rap(mm-h) run-on-projects: ['try', 'mozilla-central'] max-run-time: 1800 mozharness: extra-options: - - --test=raptor-sunspider - - --app=chrome + - --test=raptor-motionmark-htmlsuite -raptor-chrome-motionmark-htmlsuite: - description: "Raptor Chrome MotionMark HtmlSuite" - try-name: raptor-chrome-motionmark-htmlsuite +raptor-motionmark-htmlsuite-chrome: + description: "Raptor MotionMark HtmlSuite on Chrome" + try-name: raptor-motionmark-htmlsuite-chrome treeherder-symbol: Rap-C(mm-h) run-on-projects: ['try', 'mozilla-central'] max-run-time: 1800 @@ -153,9 +109,19 @@ raptor-chrome-motionmark-htmlsuite: - --test=raptor-motionmark-htmlsuite - --app=chrome -raptor-chrome-motionmark-animometer: - description: "Raptor Chrome MotionMark Animometer" - try-name: raptor-chrome-motionmark-animometer +raptor-motionmark-animometer-firefox: + description: "Raptor MotionMark Animometer on Firefox" + try-name: raptor-motionmark-animometer-firefox + treeherder-symbol: Rap(mm-a) + run-on-projects: ['try', 'mozilla-central'] + max-run-time: 1800 + mozharness: + extra-options: + - --test=raptor-motionmark-animometer + +raptor-motionmark-animometer-chrome: + description: "Raptor MotionMark Animometer on Chrome" + try-name: raptor-motionmark-animometer-chrome treeherder-symbol: Rap-C(mm-a) run-on-projects: ['try', 'mozilla-central'] max-run-time: 1800 @@ -164,9 +130,19 @@ raptor-chrome-motionmark-animometer: - --test=raptor-motionmark-animometer - --app=chrome -raptor-chrome-webaudio: - description: "Raptor Chrome WebAudio benchmark" - try-name: raptor-chrome-webaudio +raptor-webaudio-firefox: + description: "Raptor WebAudio on Firefox" + try-name: raptor-webaudio-firefox + treeherder-symbol: Rap(wa) + run-on-projects: ['try', 'mozilla-central'] + max-run-time: 1800 + mozharness: + extra-options: + - --test=raptor-webaudio + +raptor-webaudio-chrome: + description: "Raptor WebAudio on Chrome" + try-name: raptor-webaudio-chrome treeherder-symbol: Rap-C(wa) run-on-projects: ['try', 'mozilla-central'] max-run-time: 1800 @@ -174,3 +150,45 @@ raptor-chrome-webaudio: extra-options: - --test=raptor-webaudio - --app=chrome + +raptor-gdocs-firefox: + description: "Raptor gdocs on Firefox" + try-name: raptor-gdocs-firefox + treeherder-symbol: Rap(gdocs) + run-on-projects: ['try', 'mozilla-central'] + max-run-time: 1800 + mozharness: + extra-options: + - --test=raptor-gdocs + +raptor-gdocs-chrome: + description: "Raptor gdocs on Chrome" + try-name: raptor-gdocs-chrome + treeherder-symbol: Rap-C(gdocs) + run-on-projects: ['try'] + max-run-time: 1800 + mozharness: + extra-options: + - --test=raptor-gdocs + - --app=chrome + +raptor-sunspider-firefox: + description: "Raptor SunSpider on Firefox" + try-name: raptor-sunspider-firefox + treeherder-symbol: Rap(ss) + run-on-projects: ['try', 'mozilla-central'] + max-run-time: 1800 + mozharness: + extra-options: + - --test=raptor-sunspider + +raptor-sunspider-chrome: + description: "Raptor SunSpider on Chrome" + try-name: raptor-sunspider-chrome + treeherder-symbol: Rap-C(ss) + run-on-projects: ['try', 'mozilla-central'] + max-run-time: 1800 + mozharness: + extra-options: + - --test=raptor-sunspider + - --app=chrome diff --git a/taskcluster/ci/test/test-sets.yml b/taskcluster/ci/test/test-sets.yml index 82a2b3bed1b5..b0f312de4531 100644 --- a/taskcluster/ci/test/test-sets.yml +++ b/taskcluster/ci/test/test-sets.yml @@ -81,18 +81,22 @@ talos: - talos-h2 raptor: - - raptor-firefox-tp6 - - raptor-firefox-speedometer - - raptor-firefox-stylebench - - raptor-firefox-sunspider - - raptor-firefox-motionmark-htmlsuite - - raptor-firefox-motionmark-animometer - - raptor-firefox-webaudio - - raptor-chrome-tp6 - - raptor-chrome-speedometer - - raptor-chrome-stylebench - - raptor-chrome-sunspider - - raptor-chrome-webaudio + - raptor-tp6-firefox + - raptor-tp6-chrome + - raptor-speedometer-firefox + - raptor-speedometer-chrome + - raptor-stylebench-firefox + - raptor-stylebench-chrome + - raptor-motionmark-htmlsuite-firefox + - raptor-motionmark-htmlsuite-chrome + - raptor-motionmark-animometer-firefox + - raptor-motionmark-animometer-chrome + - raptor-webaudio-firefox + - raptor-webaudio-chrome + - raptor-gdocs-firefox + - raptor-gdocs-chrome + - raptor-sunspider-firefox + - raptor-sunspider-chrome awsy: - awsy diff --git a/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-gdocs.manifest b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-gdocs.manifest new file mode 100644 index 000000000000..023e5f6a98b7 --- /dev/null +++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-gdocs.manifest @@ -0,0 +1,9 @@ +[ + { + "filename": "mitmproxy-recordings-raptor-gdocs.zip", + "size": 26179496, + "digest": "eb32218815d3c2187807f3a18d6df1657e01a215820deaaa7a035f06d180780ac9652479ac7db1d8cde3728a890521aff2e0a5c4738304966b2a93ce1b7f33bb", + "algorithm": "sha512", + "unpack": true + } +] \ No newline at end of file diff --git a/testing/raptor/raptor/raptor.ini b/testing/raptor/raptor/raptor.ini index 35f9df31c3ea..6641ff962035 100644 --- a/testing/raptor/raptor/raptor.ini +++ b/testing/raptor/raptor/raptor.ini @@ -6,3 +6,4 @@ [include:tests/raptor-motionmark-htmlsuite.ini] [include:tests/raptor-motionmark-animometer.ini] [include:tests/raptor-webaudio.ini] +[include:tests/raptor-gdocs.ini] diff --git a/testing/raptor/raptor/tests/raptor-gdocs.ini b/testing/raptor/raptor/tests/raptor-gdocs.ini new file mode 100644 index 000000000000..4b458ec34644 --- /dev/null +++ b/testing/raptor/raptor/tests/raptor-gdocs.ini @@ -0,0 +1,53 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# raptor gdocs + +[DEFAULT] +type = pageload +playback = mitmproxy +playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest +python3_win_manifest = python3{x64}.manifest +playback_pageset_manifest = mitmproxy-recordings-raptor-gdocs.manifest +page_cycles = 25 +unit = ms +lower_is_better = true +alert_threshold = 2.0 +page_timeout = 30000 + +[raptor-google-docs-firefox] +apps = firefox +test_url = https://docs.google.com/document/d/1US-07msg12slQtI_xchzYxcKlTs6Fp7WqIc6W5GK5M8/edit?usp=sharing +playback_recordings = google-docs.mp +measure = fnbpaint + +[raptor-google-sheets-firefox] +apps = firefox +test_url = https://docs.google.com/spreadsheets/d/1jT9qfZFAeqNoOK97gruc34Zb7y_Q-O_drZ8kSXT-4D4/edit?usp=sharing +playback_recordings = google-sheets.mp +measure = fnbpaint + +[raptor-google-slides-firefox] +apps = firefox +test_url = https://docs.google.com/presentation/d/1Ici0ceWwpFvmIb3EmKeWSq_vAQdmmdFcWqaiLqUkJng/edit?usp=sharing +playback_recordings = google-slides.mp +measure = fnbpaint + +[raptor-google-docs-chrome] +apps = chrome +test_url = https://docs.google.com/document/d/1US-07msg12slQtI_xchzYxcKlTs6Fp7WqIc6W5GK5M8/edit?usp=sharing +playback_recordings = google-docs.mp +measure = fcp + +[raptor-google-sheets-chrome] +apps = chrome +test_url = https://docs.google.com/spreadsheets/d/1jT9qfZFAeqNoOK97gruc34Zb7y_Q-O_drZ8kSXT-4D4/edit?usp=sharing +playback_recordings = google-sheets.mp +measure = fcp + +[raptor-google-slides-chrome] +apps = chrome +test_url = https://docs.google.com/presentation/d/1Ici0ceWwpFvmIb3EmKeWSq_vAQdmmdFcWqaiLqUkJng/edit?usp=sharing +playback_recordings = google-slides.mp +measure = fcp diff --git a/testing/raptor/raptor/tests/raptor-tp6.ini b/testing/raptor/raptor/tests/raptor-tp6.ini index 6a2440e97373..d0a480e1f0b8 100644 --- a/testing/raptor/raptor/tests/raptor-tp6.ini +++ b/testing/raptor/raptor/tests/raptor-tp6.ini @@ -15,19 +15,19 @@ unit = ms lower_is_better = true alert_threshold = 2.0 -[raptor-firefox-tp6-amazon] +[raptor-tp6-amazon-firefox] apps = firefox test_url = https://www.amazon.com/s/url=search-alias%3Daps&field-keywords=laptop playback_recordings = amazon.mp measure = fnbpaint -[raptor-firefox-tp6-facebook] +[raptor-tp6-facebook-firefox] apps = firefox test_url = https://www.facebook.com playback_recordings = facebook.mp measure = fnbpaint -[raptor-firefox-tp6-google] +[raptor-tp6-google-firefox] apps = firefox # note: use the full url as the first part (without '&cad=h') redirects # to the url + '&cad=h'; that redirection causes measure.js content @@ -36,31 +36,31 @@ test_url = https://www.google.com/search?hl=en&q=barack+obama&cad=h playback_recordings = google-search.mp measure = fnbpaint -[raptor-firefox-tp6-youtube] +[raptor-tp6-youtube-firefox] apps = firefox test_url = https://www.youtube.com playback_recordings = youtube.mp measure = fnbpaint -[raptor-chrome-tp6-amazon] +[raptor-tp6-amazon-chrome] apps = chrome test_url = https://www.amazon.com/s/url=search-alias%3Daps&field-keywords=laptop playback_recordings = amazon.mp measure = fcp -[raptor-chrome-tp6-facebook] +[raptor-tp6-facebook-chrome] apps = chrome test_url = https://www.facebook.com playback_recordings = facebook.mp measure = fcp -[raptor-chrome-tp6-google] +[raptor-tp6-google-chrome] apps = chrome test_url = https://www.google.com/#hl=en&q=barack+obama playback_recordings = google-search.mp measure = fcp -[raptor-chrome-tp6-youtube] +[raptor-tp6-youtube-chrome] apps = chrome test_url = https://www.youtube.com playback_recordings = youtube.mp