From e0060de75c6bcecdefc3f0c8c2efe00874caa4f7 Mon Sep 17 00:00:00 2001 From: Ionut Goldan Date: Fri, 12 Apr 2019 10:33:24 +0000 Subject: [PATCH] Bug 1539437 - Replace Chrome with Chromium mentionings where needed r=sparky,davehunt Differential Revision: https://phabricator.services.mozilla.com/D25094 --HG-- rename : taskcluster/ci/test/raptor-chrome.yml => taskcluster/ci/test/raptor-chromium.yml extra : moz-landing-system : lando --- taskcluster/ci/config.yml | 8 +- taskcluster/ci/fetch/chromium-fetch.yml | 8 +- taskcluster/ci/test/kind.yml | 2 +- ...{raptor-chrome.yml => raptor-chromium.yml} | 210 +++++++++--------- taskcluster/ci/test/test-platforms.yml | 10 +- taskcluster/ci/test/test-sets.yml | 46 ++-- taskcluster/scripts/misc/fetch-chromium.py | 14 +- taskcluster/taskgraph/transforms/fetch.py | 6 +- .../mozbase/mozprofile/mozprofile/profile.py | 12 +- .../mozbase/mozprofile/tests/test_profile.py | 3 + .../mozrunner/mozrunner/application.py | 6 + .../mozbase/mozrunner/mozrunner/runners.py | 12 + testing/mozbase/mozrunner/tests/conftest.py | 10 +- .../configs/raptor/android_hw_config.py | 2 +- .../mozharness/configs/raptor/linux_config.py | 2 +- .../mozharness/configs/raptor/mac_config.py | 2 +- .../configs/raptor/windows_config.py | 2 +- .../configs/raptor/windows_vm_config.py | 2 +- .../mozharness/mozilla/testing/raptor.py | 65 ++++-- testing/raptor/mach_commands.py | 7 +- testing/raptor/raptor/cmdline.py | 32 ++- testing/raptor/raptor/raptor.py | 11 +- .../raptor/tests/raptor-assorted-dom.ini | 3 + .../tests/raptor-motionmark-animometer.ini | 5 +- .../tests/raptor-motionmark-htmlsuite.ini | 5 +- .../raptor/tests/raptor-speedometer.ini | 5 +- .../raptor/raptor/tests/raptor-stylebench.ini | 5 +- .../raptor/raptor/tests/raptor-sunspider.ini | 5 +- testing/raptor/raptor/tests/raptor-tp6-1.ini | 24 ++ testing/raptor/raptor/tests/raptor-tp6-10.ini | 15 ++ testing/raptor/raptor/tests/raptor-tp6-2.ini | 18 ++ testing/raptor/raptor/tests/raptor-tp6-3.ini | 19 ++ testing/raptor/raptor/tests/raptor-tp6-4.ini | 12 + testing/raptor/raptor/tests/raptor-tp6-5.ini | 12 + testing/raptor/raptor/tests/raptor-tp6-6.ini | 13 ++ testing/raptor/raptor/tests/raptor-tp6-7.ini | 20 ++ .../raptor/raptor/tests/raptor-tp6-8-404.ini | 28 +++ testing/raptor/raptor/tests/raptor-tp6-8.ini | 14 ++ testing/raptor/raptor/tests/raptor-tp6-9.ini | 14 ++ .../raptor/tests/raptor-unity-webgl.ini | 3 + .../tests/raptor-wasm-godot-baseline.ini | 2 +- .../tests/raptor-wasm-godot-cranelift.ini | 10 +- .../raptor/tests/raptor-wasm-godot-ion.ini | 2 +- .../raptor/raptor/tests/raptor-wasm-godot.ini | 5 +- .../tests/raptor-wasm-misc-baseline.ini | 2 +- .../tests/raptor-wasm-misc-cranelift.ini | 2 +- .../raptor/tests/raptor-wasm-misc-ion.ini | 2 +- .../raptor/raptor/tests/raptor-wasm-misc.ini | 5 +- .../raptor/raptor/tests/raptor-webaudio.ini | 5 +- testing/raptor/test/test_manifest.py | 21 +- testing/raptor/test/test_raptor.py | 2 + 51 files changed, 536 insertions(+), 214 deletions(-) rename taskcluster/ci/test/{raptor-chrome.yml => raptor-chromium.yml} (54%) diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml index 13a07f332a9c..450ca8f89b99 100755 --- a/taskcluster/ci/config.yml +++ b/taskcluster/ci/config.yml @@ -34,8 +34,12 @@ treeherder: 'Rap-refbrow-e10s': 'Raptor performance tests on the reference browser' 'Rap-Prof': 'Raptor performance tests on Firefox with Gecko Profiling' 'Rap-Prof-e10s': 'Raptor performance tests on Firefox with Gecko Profiling' - 'Rap-C': 'Raptor performance tests on Google Chrome' - 'Rap-C-e10s': 'Raptor performance tests on Google Chrome with e10s' + 'Rap-ChC': 'Raptor performance tests on Google Chrome Canary' + 'Rap-ChC-e10s': 'Raptor performance tests on Google Chrome Canary with e10s' + 'Rap-ChD': 'Raptor performance tests on Google Chrome Dev' + 'Rap-ChD-e10s': 'Raptor performance tests on Google Chrome Dev with e10s' + 'Rap-Cr': 'Raptor performance tests on Google Chromium' + 'Rap-Cr-e10s': 'Raptor performance tests on Google Chromium with e10s' 'Rap-P': 'Raptor power tests on Firefox' 'Rap-P-e10s': 'Raptor power tests on Firefox with e10s' 'T': 'Talos performance tests' diff --git a/taskcluster/ci/fetch/chromium-fetch.yml b/taskcluster/ci/fetch/chromium-fetch.yml index 5dac799be7ad..ed6856647fa4 100644 --- a/taskcluster/ci/fetch/chromium-fetch.yml +++ b/taskcluster/ci/fetch/chromium-fetch.yml @@ -7,22 +7,22 @@ linux64-chromium: description: 'Linux64 Chromium Fetch' fetch: platform: linux - artifact-name: chrome-linux.tar.bz2 + artifact-name: chromium-linux.tar.bz2 win32-chromium: description: 'Windows32 Chromium Fetch' fetch: platform: win32 - artifact-name: chrome-win32.tar.bz2 + artifact-name: chromium-win32.tar.bz2 win64-chromium: description: 'Windows64 Chromium Fetch' fetch: platform: win64 - artifact-name: chrome-win64.tar.bz2 + artifact-name: chromium-win64.tar.bz2 mac-chromium: description: 'MacOSX Chromium Fetch' fetch: platform: mac - artifact-name: chrome-mac.tar.bz2 + artifact-name: chromium-mac.tar.bz2 diff --git a/taskcluster/ci/test/kind.yml b/taskcluster/ci/test/kind.yml index 4208ac0e2d2a..ad7679ed248d 100644 --- a/taskcluster/ci/test/kind.yml +++ b/taskcluster/ci/test/kind.yml @@ -29,7 +29,7 @@ jobs-from: - misc.yml - mochitest.yml - raptor.yml - - raptor-chrome.yml + - raptor-chromium.yml - reftest.yml - talos.yml - web-platform.yml diff --git a/taskcluster/ci/test/raptor-chrome.yml b/taskcluster/ci/test/raptor-chromium.yml similarity index 54% rename from taskcluster/ci/test/raptor-chrome.yml rename to taskcluster/ci/test/raptor-chromium.yml index b593fcd7691f..ad70349d2b52 100644 --- a/taskcluster/ci/test/raptor-chrome.yml +++ b/taskcluster/ci/test/raptor-chromium.yml @@ -43,171 +43,171 @@ job-defaults: default: - linux64-chromium -raptor-tp6-1-chrome: - description: "Raptor tp6-1 on Chrome" - try-name: raptor-tp6-1-chrome - treeherder-symbol: Rap-C(tp6-1) +raptor-tp6-1-chromium: + description: "Raptor tp6-1 on Chromium" + try-name: raptor-tp6-1-chromium + treeherder-symbol: Rap-Cr(tp6-1) max-run-time: 1200 mozharness: extra-options: - --test=raptor-tp6-1 - - --app=chrome + - --app=chromium -raptor-tp6-2-chrome: - description: "Raptor tp6-2 on Chrome" - try-name: raptor-tp6-2-chrome - treeherder-symbol: Rap-C(tp6-2) +raptor-tp6-2-chromium: + description: "Raptor tp6-2 on Chromium" + try-name: raptor-tp6-2-chromium + treeherder-symbol: Rap-Cr(tp6-2) mozharness: extra-options: - --test=raptor-tp6-2 - - --app=chrome + - --app=chromium -raptor-tp6-3-chrome: - description: "Raptor tp6-3 on Chrome" - try-name: raptor-tp6-3-chrome - treeherder-symbol: Rap-C(tp6-3) +raptor-tp6-3-chromium: + description: "Raptor tp6-3 on Chromium" + try-name: raptor-tp6-3-chromium + treeherder-symbol: Rap-Cr(tp6-3) max-run-time: 2400 mozharness: extra-options: - --test=raptor-tp6-3 - - --app=chrome + - --app=chromium -raptor-tp6-4-chrome: - description: "Raptor tp6-4 on Chrome" - try-name: raptor-tp6-4-chrome - treeherder-symbol: Rap-C(tp6-4) +raptor-tp6-4-chromium: + description: "Raptor tp6-4 on Chromium" + try-name: raptor-tp6-4-chromium + treeherder-symbol: Rap-Cr(tp6-4) mozharness: extra-options: - --test=raptor-tp6-4 - - --app=chrome + - --app=chromium -raptor-tp6-5-chrome: - description: "Raptor tp6-5 on Chrome" - try-name: raptor-tp6-5-chrome - treeherder-symbol: Rap-C(tp6-5) +raptor-tp6-5-chromium: + description: "Raptor tp6-5 on Chromium" + try-name: raptor-tp6-5-chromium + treeherder-symbol: Rap-Cr(tp6-5) mozharness: extra-options: - --test=raptor-tp6-5 - - --app=chrome + - --app=chromium -raptor-tp6-6-chrome: - description: "Raptor tp6-6 on Chrome" - try-name: raptor-tp6-6-chrome - treeherder-symbol: Rap-C(tp6-6) +raptor-tp6-6-chromium: + description: "Raptor tp6-6 on Chromium" + try-name: raptor-tp6-6-chromium + treeherder-symbol: Rap-Cr(tp6-6) mozharness: extra-options: - --test=raptor-tp6-6 - - --app=chrome + - --app=chromium -raptor-tp6-7-chrome: - description: "Raptor tp6-7 on Chrome" - try-name: raptor-tp6-7-chrome - treeherder-symbol: Rap-C(tp6-7) +raptor-tp6-7-chromium: + description: "Raptor tp6-7 on Chromium" + try-name: raptor-tp6-7-chromium + treeherder-symbol: Rap-Cr(tp6-7) mozharness: extra-options: - --test=raptor-tp6-7 - - --app=chrome + - --app=chromium -raptor-tp6-8-chrome: - description: "Raptor tp6-8 on Chrome" - try-name: raptor-tp6-8-chrome - treeherder-symbol: Rap-C(tp6-8) +raptor-tp6-8-chromium: + description: "Raptor tp6-8 on Chromium" + try-name: raptor-tp6-8-chromium + treeherder-symbol: Rap-Cr(tp6-8) mozharness: extra-options: - --test=raptor-tp6-8 - - --app=chrome + - --app=chromium -raptor-tp6-8-404-chrome: - description: "Raptor tp6-8-404 on Chrome" - try-name: raptor-tp6-8-404-chrome - treeherder-symbol: Rap-C(tp6-8-404) +raptor-tp6-8-404-chromium: + description: "Raptor tp6-8-404 on Chromium" + try-name: raptor-tp6-8-404-chromium + treeherder-symbol: Rap-Cr(tp6-8-404) tier: 3 mozharness: extra-options: - --test=raptor-tp6-8-404 - - --app=chrome + - --app=chromium -raptor-tp6-9-chrome: - description: "Raptor tp6-9 on Chrome" - try-name: raptor-tp6-9-chrome - treeherder-symbol: Rap-C(tp6-9) +raptor-tp6-9-chromium: + description: "Raptor tp6-9 on Chromium" + try-name: raptor-tp6-9-chromium + treeherder-symbol: Rap-Cr(tp6-9) mozharness: extra-options: - --test=raptor-tp6-9 - - --app=chrome + - --app=chromium -raptor-tp6-10-chrome: - description: "Raptor tp6-10 on Chrome" - try-name: raptor-tp6-10-chrome - treeherder-symbol: Rap-C(tp6-10) +raptor-tp6-10-chromium: + description: "Raptor tp6-10 on Chromium" + try-name: raptor-tp6-10-chromium + treeherder-symbol: Rap-Cr(tp6-10) mozharness: extra-options: - --test=raptor-tp6-10 - - --app=chrome + - --app=chromium -raptor-speedometer-chrome: - description: "Raptor Speedometer on Chrome" - try-name: raptor-speedometer-chrome - treeherder-symbol: Rap-C(sp) +raptor-speedometer-chromium: + description: "Raptor Speedometer on Chromium" + try-name: raptor-speedometer-chromium + treeherder-symbol: Rap-Cr(sp) max-run-time: 1500 mozharness: extra-options: - --test=raptor-speedometer - - --app=chrome + - --app=chromium -raptor-stylebench-chrome: - description: "Raptor StyleBench on Chrome" - try-name: raptor-stylebench-chrome - treeherder-symbol: Rap-C(sb) +raptor-stylebench-chromium: + description: "Raptor StyleBench on Chromium" + try-name: raptor-stylebench-chromium + treeherder-symbol: Rap-Cr(sb) mozharness: extra-options: - --test=raptor-stylebench - - --app=chrome + - --app=chromium -raptor-motionmark-htmlsuite-chrome: - description: "Raptor MotionMark HtmlSuite on Chrome" - try-name: raptor-motionmark-htmlsuite-chrome - treeherder-symbol: Rap-C(mm-h) +raptor-motionmark-htmlsuite-chromium: + description: "Raptor MotionMark HtmlSuite on Chromium" + try-name: raptor-motionmark-htmlsuite-chromium + treeherder-symbol: Rap-Cr(mm-h) mozharness: extra-options: - --test=raptor-motionmark-htmlsuite - - --app=chrome + - --app=chromium -raptor-motionmark-animometer-chrome: - description: "Raptor MotionMark Animometer on Chrome" - try-name: raptor-motionmark-animometer-chrome - treeherder-symbol: Rap-C(mm-a) +raptor-motionmark-animometer-chromium: + description: "Raptor MotionMark Animometer on Chromium" + try-name: raptor-motionmark-animometer-chromium + treeherder-symbol: Rap-Cr(mm-a) mozharness: extra-options: - --test=raptor-motionmark-animometer - - --app=chrome + - --app=chromium -raptor-webaudio-chrome: - description: "Raptor WebAudio on Chrome" - try-name: raptor-webaudio-chrome - treeherder-symbol: Rap-C(wa) +raptor-webaudio-chromium: + description: "Raptor WebAudio on Chromium" + try-name: raptor-webaudio-chromium + treeherder-symbol: Rap-Cr(wa) mozharness: extra-options: - --test=raptor-webaudio - - --app=chrome + - --app=chromium -raptor-sunspider-chrome: - description: "Raptor SunSpider on Chrome" - try-name: raptor-sunspider-chrome - treeherder-symbol: Rap-C(ss) +raptor-sunspider-chromium: + description: "Raptor SunSpider on Chromium" + try-name: raptor-sunspider-chromium + treeherder-symbol: Rap-Cr(ss) mozharness: extra-options: - --test=raptor-sunspider - - --app=chrome + - --app=chromium -raptor-unity-webgl-chrome: - description: "Raptor Unity WebGL on Chrome" - try-name: raptor-unity-webgl-chrome - treeherder-symbol: Rap-C(ugl) +raptor-unity-webgl-chromium: + description: "Raptor Unity WebGL on Chromium" + try-name: raptor-unity-webgl-chromium + treeherder-symbol: Rap-Cr(ugl) mozharness: extra-options: - --test=raptor-unity-webgl - - --app=chrome + - --app=chromium fetches: fetch: by-test-platform: @@ -224,14 +224,14 @@ raptor-unity-webgl-chrome: - linux64-chromium - unity-webgl -raptor-wasm-misc-chrome: - description: "Raptor WASM Misc on Chrome" - try-name: raptor-wasm-misc-chrome - treeherder-symbol: Rap-C(wm) +raptor-wasm-misc-chromium: + description: "Raptor WASM Misc on Chromium" + try-name: raptor-wasm-misc-chromium + treeherder-symbol: Rap-Cr(wm) mozharness: extra-options: - --test=raptor-wasm-misc - - --app=chrome + - --app=chromium fetches: fetch: by-test-platform: @@ -248,15 +248,15 @@ raptor-wasm-misc-chrome: - linux64-chromium - wasm-misc -raptor-assorted-dom-chrome: - description: "Raptor Assorted-Dom on Chrome" - try-name: raptor-assorted-dom-chrome - treeherder-symbol: Rap-C(dom) +raptor-assorted-dom-chromium: + description: "Raptor Assorted-Dom on Chromium" + try-name: raptor-assorted-dom-chromium + treeherder-symbol: Rap-Cr(dom) max-run-time: 1500 mozharness: extra-options: - --test=raptor-assorted-dom - - --app=chrome + - --app=chromium fetches: fetch: by-test-platform: @@ -273,12 +273,12 @@ raptor-assorted-dom-chrome: - linux64-chromium - assorted-dom -raptor-wasm-godot-chrome: - description: "Raptor Wasm Godot on Chrome" - try-name: raptor-wasm-godot-chrome - treeherder-symbol: Rap-C(godot) +raptor-wasm-godot-chromium: + description: "Raptor Wasm Godot on Chromium" + try-name: raptor-wasm-godot-chromium + treeherder-symbol: Rap-Cr(godot) max-run-time: 1500 mozharness: extra-options: - --test=raptor-wasm-godot - - --app=chrome + - --app=chromium diff --git a/taskcluster/ci/test/test-platforms.yml b/taskcluster/ci/test/test-platforms.yml index 67f54e4b2e26..32a282ee34fb 100644 --- a/taskcluster/ci/test/test-platforms.yml +++ b/taskcluster/ci/test/test-platforms.yml @@ -62,8 +62,8 @@ linux64-shippable/opt: - awsy - mochitest-headless - linux-talos-flex - - raptor-chrome - - raptor-fetch-chrome + - raptor-chromium + - raptor-fetch-chromium - raptor-firefox - raptor-fetch-firefox - raptor-profiling @@ -158,7 +158,7 @@ windows7-32-shippable/opt: - windows-talos - windows-tests - web-platform-tests - - raptor-chrome + - raptor-chromium - raptor-firefox windows7-32-devedition/opt: @@ -230,7 +230,7 @@ windows10-64-shippable/opt: - web-platform-tests - mochitest-headless - raptor-firefox - - raptor-chrome + - raptor-chromium - raptor-profiling windows10-64-devedition/opt: @@ -287,7 +287,7 @@ macosx64-shippable/opt: - macosx64-tests - desktop-screenshot-capture - awsy - - raptor-chrome + - raptor-chromium - raptor-firefox - raptor-profiling - marionette-media-tests diff --git a/taskcluster/ci/test/test-sets.yml b/taskcluster/ci/test/test-sets.yml index 540ea5d0d118..8f062f985b00 100644 --- a/taskcluster/ci/test/test-sets.yml +++ b/taskcluster/ci/test/test-sets.yml @@ -119,25 +119,25 @@ raptor-profiling: - raptor-sunspider-firefox-profiling - raptor-wasm-godot-firefox-profiling -raptor-chrome: - - raptor-tp6-1-chrome - - raptor-tp6-2-chrome - - raptor-tp6-3-chrome - - raptor-tp6-4-chrome - - raptor-tp6-5-chrome - - raptor-tp6-6-chrome - - raptor-tp6-7-chrome - - raptor-tp6-8-chrome - - raptor-tp6-8-404-chrome - - raptor-tp6-9-chrome - - raptor-tp6-10-chrome - - raptor-speedometer-chrome - - raptor-stylebench-chrome - - raptor-motionmark-htmlsuite-chrome - - raptor-motionmark-animometer-chrome - - raptor-webaudio-chrome - - raptor-sunspider-chrome - - raptor-wasm-godot-chrome +raptor-chromium: + - raptor-tp6-1-chromium + - raptor-tp6-2-chromium + - raptor-tp6-3-chromium + - raptor-tp6-4-chromium + - raptor-tp6-5-chromium + - raptor-tp6-6-chromium + - raptor-tp6-7-chromium + - raptor-tp6-8-chromium + - raptor-tp6-8-404-chromium + - raptor-tp6-9-chromium + - raptor-tp6-10-chromium + - raptor-speedometer-chromium + - raptor-stylebench-chromium + - raptor-motionmark-htmlsuite-chromium + - raptor-motionmark-animometer-chromium + - raptor-webaudio-chromium + - raptor-sunspider-chromium + - raptor-wasm-godot-chromium # Fetch tasks are only supported on Linux for now, # so these need to be separate sets. @@ -166,10 +166,10 @@ raptor-fetch-geckoview: - raptor-unity-webgl-geckoview - raptor-unity-webgl-refbrow -raptor-fetch-chrome: - - raptor-unity-webgl-chrome - - raptor-wasm-misc-chrome - - raptor-assorted-dom-chrome +raptor-fetch-chromium: + - raptor-unity-webgl-chromium + - raptor-wasm-misc-chromium + - raptor-assorted-dom-chromium awsy: - awsy diff --git a/taskcluster/scripts/misc/fetch-chromium.py b/taskcluster/scripts/misc/fetch-chromium.py index 01d19207a56a..ca7e408755ea 100644 --- a/taskcluster/scripts/misc/fetch-chromium.py +++ b/taskcluster/scripts/misc/fetch-chromium.py @@ -33,22 +33,22 @@ CHROMIUM_INFO = { 'linux': { 'platform': 'Linux_x64', 'archive': 'chrome-linux.zip', - 'result': 'chrome-linux.tar.bz2' + 'result': 'chromium-linux.tar.bz2' }, 'win32': { 'platform': 'Win_x64', 'archive': 'chrome-win.zip', - 'result': 'chrome-win32.tar.bz2' + 'result': 'chromium-win32.tar.bz2' }, 'win64': { 'platform': 'Win_x64', 'archive': 'chrome-win.zip', - 'result': 'chrome-win64.tar.bz2' + 'result': 'chromium-win64.tar.bz2' }, 'mac': { 'platform': 'Mac', 'archive': 'chrome-mac.zip', - 'result': 'chrome-mac.tar.bz2' + 'result': 'chromium-mac.tar.bz2' } } @@ -137,7 +137,7 @@ def build_chromium_archive(platform, revision=None): # Make a temporary location for the file tmppath = tempfile.mkdtemp() - tmpzip = os.path.join(tmppath, 'tmp-chrome.zip') + tmpzip = os.path.join(tmppath, 'tmp-chromium.zip') revision = fetch_chromium_build(platform, revision, tmpzip) @@ -160,8 +160,8 @@ def build_chromium_archive(platform, revision=None): "Could not find any directories after extraction of chromium zip." ) - chrome_dir = os.path.join(tmppath, dirs[0]) - revision_file = os.path.join(chrome_dir, '.REVISION') + chromium_dir = os.path.join(tmppath, dirs[0]) + revision_file = os.path.join(chromium_dir, '.REVISION') with open(revision_file, 'w+') as f: f.write(str(revision)) diff --git a/taskcluster/taskgraph/transforms/fetch.py b/taskcluster/taskgraph/transforms/fetch.py index 7ff9c214bd54..24523ae7171e 100644 --- a/taskcluster/taskgraph/transforms/fetch.py +++ b/taskcluster/taskgraph/transforms/fetch.py @@ -258,7 +258,11 @@ def create_chromium_fetch_task(config, job): task, cache_type=CACHE_TYPE, cache_name=cache_name, - digest_data=["revision={}".format(revision), "platform={}".format(platform)], + digest_data=[ + "revision={}".format(revision), + "platform={}".format(platform), + "artifact_name={}".format(artifact_name), + ], ) return task diff --git a/testing/mozbase/mozprofile/mozprofile/profile.py b/testing/mozbase/mozprofile/mozprofile/profile.py index 761560488dcf..705ce0f61427 100644 --- a/testing/mozbase/mozprofile/mozprofile/profile.py +++ b/testing/mozbase/mozprofile/mozprofile/profile.py @@ -22,6 +22,7 @@ from .prefs import Preferences __all__ = ['BaseProfile', 'ChromeProfile', + 'ChromiumProfile', 'Profile', 'FirefoxProfile', 'ThunderbirdProfile', @@ -448,7 +449,7 @@ class ThunderbirdProfile(Profile): } -class ChromeProfile(BaseProfile): +class ChromiumProfile(BaseProfile): preference_file_names = ('Preferences',) class AddonManager(list): @@ -463,7 +464,7 @@ class ChromeProfile(BaseProfile): return False def __init__(self, **kwargs): - super(ChromeProfile, self).__init__(**kwargs) + super(ChromiumProfile, self).__init__(**kwargs) if self.create_new: self.profile = os.path.join(self.profile, 'Default') @@ -495,8 +496,15 @@ class ChromeProfile(BaseProfile): fh.write(prefstr) +class ChromeProfile(ChromiumProfile): + # update this if Google Chrome requires more + # specific profiles + pass + + profile_class = { 'chrome': ChromeProfile, + 'chromium': ChromiumProfile, 'firefox': FirefoxProfile, 'thunderbird': ThunderbirdProfile, } diff --git a/testing/mozbase/mozprofile/tests/test_profile.py b/testing/mozbase/mozprofile/tests/test_profile.py index b5821f554ec9..4437c9ffe542 100644 --- a/testing/mozbase/mozprofile/tests/test_profile.py +++ b/testing/mozbase/mozprofile/tests/test_profile.py @@ -16,6 +16,7 @@ from mozprofile import ( BaseProfile, Profile, ChromeProfile, + ChromiumProfile, FirefoxProfile, ThunderbirdProfile, create_profile, @@ -44,6 +45,7 @@ def test_with_profile_should_cleanup_even_on_exception(): @pytest.mark.parametrize('app,cls', [ ('chrome', ChromeProfile), + ('chromium', ChromiumProfile), ('firefox', FirefoxProfile), ('thunderbird', ThunderbirdProfile), ('unknown', None) @@ -65,6 +67,7 @@ def test_create_profile(tmpdir, app, cls): @pytest.mark.parametrize('cls', [ Profile, ChromeProfile, + ChromiumProfile, ]) def test_merge_profile(cls): profile = cls(preferences={'foo': 'bar'}) diff --git a/testing/mozbase/mozrunner/mozrunner/application.py b/testing/mozbase/mozrunner/mozrunner/application.py index a17ad55e5f11..744225f52180 100644 --- a/testing/mozbase/mozrunner/mozrunner/application.py +++ b/testing/mozbase/mozrunner/mozrunner/application.py @@ -13,6 +13,7 @@ from mozdevice import ADBDevice from mozprofile import ( Profile, ChromeProfile, + ChromiumProfile, FirefoxProfile, ThunderbirdProfile ) @@ -23,6 +24,7 @@ here = os.path.abspath(os.path.dirname(__file__)) def get_app_context(appname): context_map = { 'chrome': ChromeContext, + 'chromium': ChromiumContext, 'default': DefaultContext, 'fennec': FennecContext, 'firefox': FirefoxContext, @@ -148,3 +150,7 @@ class ThunderbirdContext(object): class ChromeContext(object): profile_class = ChromeProfile + + +class ChromiumContext(object): + profile_class = ChromiumProfile diff --git a/testing/mozbase/mozrunner/mozrunner/runners.py b/testing/mozbase/mozrunner/mozrunner/runners.py index a1fafec6e566..19796232f3bd 100644 --- a/testing/mozbase/mozrunner/mozrunner/runners.py +++ b/testing/mozbase/mozrunner/mozrunner/runners.py @@ -85,6 +85,17 @@ def ChromeRunner(*args, **kwargs): return BlinkRuntimeRunner(*args, **kwargs) +def ChromiumRunner(*args, **kwargs): + """ + Create a desktop Google Chromium runner. + + :param binary: Path to Chromium binary. + :param cmdargs: Arguments to pass into the binary. + """ + kwargs['app_ctx'] = get_app_context('chromium')() + return BlinkRuntimeRunner(*args, **kwargs) + + def FennecEmulatorRunner(avd='mozemulator-4.3', adb_path=None, avd_home=None, @@ -125,6 +136,7 @@ def FennecEmulatorRunner(avd='mozemulator-4.3', runners = { 'chrome': ChromeRunner, + 'chromium': ChromiumRunner, 'default': Runner, 'firefox': FirefoxRunner, 'fennec': FennecEmulatorRunner, diff --git a/testing/mozbase/mozrunner/tests/conftest.py b/testing/mozbase/mozrunner/tests/conftest.py index a5a940053391..ad6ac86053b9 100644 --- a/testing/mozbase/mozrunner/tests/conftest.py +++ b/testing/mozbase/mozrunner/tests/conftest.py @@ -19,13 +19,15 @@ def get_binary(): os.environ['GECKO_BINARY_PATH'] = os.environ['BROWSER_PATH'] def inner(app): - if app not in ('chrome', 'firefox'): + if app not in ('chrome', 'chromium', 'firefox'): pytest.xfail(reason="{} support not implemented".format(app)) if app == 'firefox': binary = fixtures.binary() elif app == 'chrome': binary = os.environ.get('CHROME_BINARY_PATH') + elif app == 'chromium': + binary = os.environ.get('CHROMIUM_BINARY_PATH') if not binary: pytest.skip("could not find a {} binary".format(app)) @@ -33,14 +35,14 @@ def get_binary(): return inner -@pytest.fixture(params=['firefox', 'chrome']) +@pytest.fixture(params=['firefox', 'chrome', 'chromium']) def runner(request, get_binary): app = request.param binary = get_binary(app) cmdargs = ['--headless'] - if app == 'chrome': - # prevents headless chrome from exiting after loading the page + if app in ['chrome', 'chromium']: + # prevents headless chromium from exiting after loading the page cmdargs.append('--remote-debugging-port=9222') # only needed on Windows, but no harm in specifying it everywhere cmdargs.append('--disable-gpu') diff --git a/testing/mozharness/configs/raptor/android_hw_config.py b/testing/mozharness/configs/raptor/android_hw_config.py index 4b3c0192d180..b0550bc4244a 100644 --- a/testing/mozharness/configs/raptor/android_hw_config.py +++ b/testing/mozharness/configs/raptor/android_hw_config.py @@ -7,7 +7,7 @@ config = { "clobber", "download-and-extract", "populate-webroot", - "install-chrome", + "install-chromium-distribution", "create-virtualenv", "install", "run-tests", diff --git a/testing/mozharness/configs/raptor/linux_config.py b/testing/mozharness/configs/raptor/linux_config.py index 01a49f7cd148..f21ef47cdc58 100644 --- a/testing/mozharness/configs/raptor/linux_config.py +++ b/testing/mozharness/configs/raptor/linux_config.py @@ -18,7 +18,7 @@ config = { "clobber", "download-and-extract", "populate-webroot", - "install-chrome", + "install-chromium-distribution", "create-virtualenv", "install", "run-tests", diff --git a/testing/mozharness/configs/raptor/mac_config.py b/testing/mozharness/configs/raptor/mac_config.py index 3dbe31208a44..6b8f4d41d1a3 100644 --- a/testing/mozharness/configs/raptor/mac_config.py +++ b/testing/mozharness/configs/raptor/mac_config.py @@ -21,7 +21,7 @@ config = { "clobber", "download-and-extract", "populate-webroot", - "install-chrome", + "install-chromium-distribution", "create-virtualenv", "install", "run-tests", diff --git a/testing/mozharness/configs/raptor/windows_config.py b/testing/mozharness/configs/raptor/windows_config.py index bdde0d496a89..dcee65610b4b 100644 --- a/testing/mozharness/configs/raptor/windows_config.py +++ b/testing/mozharness/configs/raptor/windows_config.py @@ -25,7 +25,7 @@ config = { "clobber", "download-and-extract", "populate-webroot", - "install-chrome", + "install-chromium-distribution", "create-virtualenv", "install", "run-tests", diff --git a/testing/mozharness/configs/raptor/windows_vm_config.py b/testing/mozharness/configs/raptor/windows_vm_config.py index 52616d07c7c6..9aa6758d4e08 100644 --- a/testing/mozharness/configs/raptor/windows_vm_config.py +++ b/testing/mozharness/configs/raptor/windows_vm_config.py @@ -24,7 +24,7 @@ config = { "clobber", "download-and-extract", "populate-webroot", - "install-chrome", + "install-chromium-distribution", "create-virtualenv", "install", "run-tests", diff --git a/testing/mozharness/mozharness/mozilla/testing/raptor.py b/testing/mozharness/mozharness/mozilla/testing/raptor.py index 17aaed5845be..f29c7d3e8516 100644 --- a/testing/mozharness/mozharness/mozilla/testing/raptor.py +++ b/testing/mozharness/mozharness/mozilla/testing/raptor.py @@ -56,7 +56,7 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin): }], [["--app"], {"default": "firefox", - "choices": ["firefox", "chrome", "geckoview", "fennec", "refbrow", "fenix"], + "choices": ["firefox", "chrome", "chromium", "fennec", "geckoview", "refbrow", "fenix"], "dest": "app", "help": "name of the application we are testing (default: firefox)" }], @@ -158,7 +158,7 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin): kwargs.setdefault('all_actions', ['clobber', 'download-and-extract', 'populate-webroot', - 'install-chrome', + 'install-chromium-distribution', 'create-virtualenv', 'install', 'run-tests', @@ -166,7 +166,7 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin): kwargs.setdefault('default_actions', ['clobber', 'download-and-extract', 'populate-webroot', - 'install-chrome', + 'install-chromium-distribution', 'create-virtualenv', 'install', 'run-tests', @@ -258,38 +258,59 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin): self.abs_dirs = abs_dirs return self.abs_dirs - def install_chrome(self): - '''install google chrome in production; installation + def install_chromium_distribution(self): + '''install Google Chromium distribution in production; installation requirements depend on the platform''' - if self.app != "chrome": - self.info("Google Chrome is not required") + linux, mac, win = 'linux', 'mac', 'win' + chrome, chromium = 'chrome', 'chromium' + + available_chromium_dists = [chrome, chromium] + binary_location = { + chromium: { + linux: ['chrome-linux', 'chrome'], + mac: ['chrome-mac', 'Chromium.app', 'Contents', 'MacOS', 'Chromium'], + win: ['chrome-win', 'Chrome.exe'] + }, + } + + if self.app == "chrome": + # remove this condition when integrating Google Chrome + raise ValueError("Google Chrome not currently integrated!") + + if self.app not in available_chromium_dists: + self.info("Google Chrome or Chromium distributions are not required.") return + chromium_dist = self.app + if self.config.get("run_local"): - self.info("expecting Google Chrome to be pre-installed locally") + self.info("expecting %s to be pre-installed locally" % chromium_dist) return - self.info("Getting fetched chromium build") - self.chrome_dest = os.path.normpath(os.path.abspath(os.environ['MOZ_FETCHES_DIR'])) + self.info("Getting fetched %s build" % chromium_dist) + self.chromium_dist_dest = os.path.normpath(os.path.abspath(os.environ['MOZ_FETCHES_DIR'])) - if 'mac' in self.platform_name(): - self.chrome_path = os.path.join(self.chrome_dest, 'chrome-mac', 'Chromium.app', - 'Contents', 'MacOS', 'Chromium') + if mac in self.platform_name(): + self.chromium_dist_path = os.path.join(self.chromium_dist_dest, + *binary_location[chromium_dist][mac]) - elif 'linux' in self.platform_name(): - self.chrome_path = os.path.join(self.chrome_dest, 'chrome-linux', 'chrome') + elif linux in self.platform_name(): + self.chromium_dist_path = os.path.join(self.chromium_dist_dest, + *binary_location[chromium_dist][linux]) else: - self.chrome_path = os.path.join(self.chrome_dest, 'chrome-win', 'Chrome.exe') + self.chromium_dist_path = os.path.join(self.chromium_dist_dest, + *binary_location[chromium_dist][win]) - self.info("chrome dest is: %s" % self.chrome_dest) - self.info("chrome path is: %s" % self.chrome_path) + self.info("%s dest is: %s" % (chromium_dist, self.chromium_dist_dest)) + self.info("%s path is: %s" % (chromium_dist, self.chromium_dist_path)) # now ensure chrome binary exists - if os.path.exists(self.chrome_path): - self.info("successfully installed Google Chrome to: %s" % self.chrome_path) + if os.path.exists(self.chromium_dist_path): + self.info("successfully installed %s to: %s" + % (chromium_dist, self.chromium_dist_path)) else: - self.info("abort: failed to install Google Chrome") + self.info("abort: failed to install %s" % chromium_dist) def raptor_options(self, args=None, **kw): """return options to raptor""" @@ -314,7 +335,7 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin): if not self.run_local: # when running locally we already set the chrome binary above in init; here # in production we aready installed chrome, so set the binary path to our install - kw_options['binary'] = self.chrome_path + kw_options['binary'] = self.chromium_dist_path # options overwritten from **kw if 'test' in self.config: diff --git a/testing/raptor/mach_commands.py b/testing/raptor/mach_commands.py index 2af865486630..0702e30341a9 100644 --- a/testing/raptor/mach_commands.py +++ b/testing/raptor/mach_commands.py @@ -114,7 +114,12 @@ class RaptorRunner(MozbuildObject): shutil.copytree(path, dest) def make_config(self): - default_actions = ['populate-webroot', 'install-chrome', 'create-virtualenv', 'run-tests'] + default_actions = [ + 'populate-webroot', + 'install-chromium-distribution', + 'create-virtualenv', + 'run-tests' + ] self.config = { 'run_local': True, 'binary_path': self.binary_path, diff --git a/testing/raptor/raptor/cmdline.py b/testing/raptor/raptor/cmdline.py index 9a2a5a8cac59..099169f282d5 100644 --- a/testing/raptor/raptor/cmdline.py +++ b/testing/raptor/raptor/cmdline.py @@ -8,24 +8,34 @@ import os from mozlog.commandline import add_logging_group - +(FIREFOX, + CHROME, + CHROMIUM) = DESKTOP_APPS = ["firefox", "chrome", "chromium"] +(FENNEC, + GECKOVIEW, + REFBROW, + FENIX) = FIREFOX_ANDROID_APPS = ["fennec", "geckoview", "refbrow", "fenix"] +CHROMIUM_DISTROS = [CHROME, CHROMIUM] APPS = { - "firefox": { + FIREFOX: { "long_name": "Firefox Desktop"}, - "chrome": { + CHROME: { "long_name": "Google Chrome Desktop"}, - "fennec": { + CHROMIUM: { + "long_name": "Google Chromium Desktop"}, + FENNEC: { "long_name": "Firefox Fennec on Android"}, - "geckoview": { + GECKOVIEW: { "long_name": "Firefox Geckoview on Android", "default_activity": "GeckoViewActivity"}, - "refbrow": { + REFBROW: { "long_name": "Firefox Android Components Reference Browser", "default_activity": "BrowserTestActivity"}, - "fenix": { + FENIX: { "long_name": "Firefox Android Fenix Browser", "default_activity": "HomeActivity"} } +INTEGRATED_APPS = list(APPS.keys()) def print_all_activities(): @@ -114,7 +124,7 @@ def verify_options(parser, args): parser.error("--binary is required!") # if running on a desktop browser make sure the binary exists - if args.app in ["firefox", "chrome"]: + if args.app in DESKTOP_APPS: if not os.path.isfile(args.binary): parser.error("{binary} does not exist!".format(**ctx)) @@ -161,13 +171,17 @@ class _StopAction(argparse.Action): class _PrintTests(_StopAction): + def __init__(self, integrated_apps=INTEGRATED_APPS, *args, **kwargs): + super(_PrintTests, self).__init__(*args, **kwargs) + self.integrated_apps = integrated_apps + def __call__(self, parser, namespace, values, option_string=None): from manifestparser import TestManifest here = os.path.abspath(os.path.dirname(__file__)) raptor_ini = os.path.join(here, 'raptor.ini') - for _app in ["firefox", "chrome", "fennec", "geckoview", "refbrow", "fenix"]: + for _app in self.integrated_apps: test_manifest = TestManifest([raptor_ini], strict=False) info = {"app": _app} available_tests = test_manifest.active_tests(exists=False, diff --git a/testing/raptor/raptor/raptor.py b/testing/raptor/raptor/raptor.py index 41a739b1cda5..4967efb49dc8 100644 --- a/testing/raptor/raptor/raptor.py +++ b/testing/raptor/raptor/raptor.py @@ -46,7 +46,9 @@ except ImportError: build = None from benchmark import Benchmark -from cmdline import parse_args +from cmdline import (parse_args, + FIREFOX_ANDROID_APPS, + CHROMIUM_DISTROS) from control_server import RaptorControlServer from gen_test_config import gen_test_config from outputhandler import OutputHandler @@ -93,7 +95,7 @@ class Raptor(object): self.post_startup_delay = post_startup_delay self.device = None self.profile_class = app - self.firefox_android_apps = ['fennec', 'geckoview', 'refbrow', 'fenix'] + self.firefox_android_apps = FIREFOX_ANDROID_APPS # debug mode is currently only supported when running locally self.debug_mode = debug_mode if self.config['run_local'] else False @@ -241,7 +243,8 @@ class Raptor(object): self.profile.addons.remove_addon(self.webext_id) # for chrome the addon is just a list (appended to cmd line) - if self.config['app'] in ["chrome", "chrome-android"]: + chrome_apps = CHROMIUM_DISTROS + ["chrome-android", "chromium-android"] + if self.config['app'] in chrome_apps: self.profile.addons.remove(self.raptor_webext) def set_test_browser_prefs(self, test_prefs): @@ -829,7 +832,7 @@ def main(args=sys.argv[1:]): if args.app == "firefox": raptor_class = RaptorDesktopFirefox - elif args.app == "chrome": + elif args.app in CHROMIUM_DISTROS: raptor_class = RaptorDesktopChrome else: raptor_class = RaptorAndroid diff --git a/testing/raptor/raptor/tests/raptor-assorted-dom.ini b/testing/raptor/raptor/tests/raptor-assorted-dom.ini index 74485282749e..6f3798b3c9b2 100644 --- a/testing/raptor/raptor/tests/raptor-assorted-dom.ini +++ b/testing/raptor/raptor/tests/raptor-assorted-dom.ini @@ -14,3 +14,6 @@ apps = firefox [raptor-assorted-dom-chrome] apps = chrome + +[raptor-assorted-dom-chromium] +apps = chromium diff --git a/testing/raptor/raptor/tests/raptor-motionmark-animometer.ini b/testing/raptor/raptor/tests/raptor-motionmark-animometer.ini index a2b80a809e1d..eb01cc97730c 100644 --- a/testing/raptor/raptor/tests/raptor-motionmark-animometer.ini +++ b/testing/raptor/raptor/tests/raptor-motionmark-animometer.ini @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -# motionmark benchmark for firefox and chrome +# motionmark benchmark for firefox and chromium distributions [DEFAULT] type = benchmark @@ -20,3 +20,6 @@ apps = firefox [raptor-motionmark-animometer-chrome] apps = chrome + +[raptor-motionmark-animometer-chromium] +apps = chromium diff --git a/testing/raptor/raptor/tests/raptor-motionmark-htmlsuite.ini b/testing/raptor/raptor/tests/raptor-motionmark-htmlsuite.ini index 5aa979bdcedb..d71aeae946fb 100644 --- a/testing/raptor/raptor/tests/raptor-motionmark-htmlsuite.ini +++ b/testing/raptor/raptor/tests/raptor-motionmark-htmlsuite.ini @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -# motionmark benchmark for firefox and chrome +# motionmark benchmark for firefox and chromium distributions [DEFAULT] type = benchmark @@ -20,3 +20,6 @@ apps = firefox [raptor-motionmark-htmlsuite-chrome] apps = chrome + +[raptor-motionmark-htmlsuite-chromium] +apps = chromium diff --git a/testing/raptor/raptor/tests/raptor-speedometer.ini b/testing/raptor/raptor/tests/raptor-speedometer.ini index d73e9feb093c..d896c536052a 100644 --- a/testing/raptor/raptor/tests/raptor-speedometer.ini +++ b/testing/raptor/raptor/tests/raptor-speedometer.ini @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -# speedometer benchmark for firefox and chrome +# speedometer benchmark for firefox and chromium distributions [DEFAULT] type = benchmark @@ -23,6 +23,9 @@ apps = firefox [raptor-speedometer-chrome] apps = chrome +[raptor-speedometer-chromium] +apps = chromium + [raptor-speedometer-geckoview] page_timeout = 900000 # 15 min page_cycles = 1 diff --git a/testing/raptor/raptor/tests/raptor-stylebench.ini b/testing/raptor/raptor/tests/raptor-stylebench.ini index 4deb1da67dbf..c1a5ea675635 100644 --- a/testing/raptor/raptor/tests/raptor-stylebench.ini +++ b/testing/raptor/raptor/tests/raptor-stylebench.ini @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -# speedometer benchmark for firefox and chrome +# speedometer benchmark for firefox and chromium distributions [DEFAULT] type = benchmark @@ -20,3 +20,6 @@ apps = firefox [raptor-stylebench-chrome] apps = chrome + +[raptor-stylebench-chromium] +apps = chromium diff --git a/testing/raptor/raptor/tests/raptor-sunspider.ini b/testing/raptor/raptor/tests/raptor-sunspider.ini index a4f8a07cec7b..e8a00157e3d4 100644 --- a/testing/raptor/raptor/tests/raptor-sunspider.ini +++ b/testing/raptor/raptor/tests/raptor-sunspider.ini @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -# sunspider benchmark for firefox and chrome +# sunspider benchmark for firefox and chromium distributions [DEFAULT] type = benchmark @@ -20,3 +20,6 @@ apps = firefox [raptor-sunspider-chrome] apps = chrome + +[raptor-sunspider-chromium] +apps = chromium diff --git a/testing/raptor/raptor/tests/raptor-tp6-1.ini b/testing/raptor/raptor/tests/raptor-tp6-1.ini index 7c7350081045..a7adafac89df 100644 --- a/testing/raptor/raptor/tests/raptor-tp6-1.ini +++ b/testing/raptor/raptor/tests/raptor-tp6-1.ini @@ -69,3 +69,27 @@ apps = chrome test_url = https://www.youtube.com playback_recordings = youtube.mp measure = fcp, loadtime + +[raptor-tp6-amazon-chromium] +apps = chromium +test_url = https://www.amazon.com/s/url=search-alias%3Daps&field-keywords=laptop +playback_recordings = amazon.mp +measure = fcp, loadtime + +[raptor-tp6-facebook-chromium] +apps = chromium +test_url = https://www.facebook.com +playback_recordings = facebook.mp +measure = fcp, loadtime + +[raptor-tp6-google-chromium] +apps = chromium +test_url = https://www.google.com/#hl=en&q=barack+obama +playback_recordings = google-search.mp +measure = fcp, loadtime + +[raptor-tp6-youtube-chromium] +apps = chromium +test_url = https://www.youtube.com +playback_recordings = youtube.mp +measure = fcp, loadtime diff --git a/testing/raptor/raptor/tests/raptor-tp6-10.ini b/testing/raptor/raptor/tests/raptor-tp6-10.ini index f7f70ad6ae03..de03d8aaaa7e 100644 --- a/testing/raptor/raptor/tests/raptor-tp6-10.ini +++ b/testing/raptor/raptor/tests/raptor-tp6-10.ini @@ -45,3 +45,18 @@ test_url = https://www.tumblr.com/dashboard playback_pageset_manifest = mitmproxy-recordings-raptor-tumblr.manifest playback_recordings = tumblr.mp measure = fcp, loadtime + +[raptor-tp6-paypal-chromium] +apps = chromium +test_url = https://www.paypal.com/myaccount/summary/ +playback_pageset_manifest = mitmproxy-recordings-raptor-paypal.manifest +playback_recordings = paypal.mp +measure = fcp, loadtime +disabled = Bug 1542757 - Investigate and fix TEST-UNEXPECTED-FAIL: test 'raptor-tp6-paypal-chrome' + +[raptor-tp6-tumblr-chromium] +apps = chromium +test_url = https://www.tumblr.com/dashboard +playback_pageset_manifest = mitmproxy-recordings-raptor-tumblr.manifest +playback_recordings = tumblr.mp +measure = fcp, loadtime diff --git a/testing/raptor/raptor/tests/raptor-tp6-2.ini b/testing/raptor/raptor/tests/raptor-tp6-2.ini index 29415ff1eff2..bce8c210cd61 100644 --- a/testing/raptor/raptor/tests/raptor-tp6-2.ini +++ b/testing/raptor/raptor/tests/raptor-tp6-2.ini @@ -53,3 +53,21 @@ apps = chrome test_url = https://docs.google.com/presentation/d/1Ici0ceWwpFvmIb3EmKeWSq_vAQdmmdFcWqaiLqUkJng/edit?usp=sharing playback_recordings = google-slides.mp measure = fcp, loadtime + +[raptor-tp6-docs-chromium] +apps = chromium +test_url = https://docs.google.com/document/d/1US-07msg12slQtI_xchzYxcKlTs6Fp7WqIc6W5GK5M8/edit?usp=sharing +playback_recordings = google-docs.mp +measure = fcp, loadtime + +[raptor-tp6-sheets-chromium] +apps = chromium +test_url = https://docs.google.com/spreadsheets/d/1jT9qfZFAeqNoOK97gruc34Zb7y_Q-O_drZ8kSXT-4D4/edit?usp=sharing +playback_recordings = google-sheets.mp +measure = fcp, loadtime + +[raptor-tp6-slides-chromium] +apps = chromium +test_url = https://docs.google.com/presentation/d/1Ici0ceWwpFvmIb3EmKeWSq_vAQdmmdFcWqaiLqUkJng/edit?usp=sharing +playback_recordings = google-slides.mp +measure = fcp, loadtime diff --git a/testing/raptor/raptor/tests/raptor-tp6-3.ini b/testing/raptor/raptor/tests/raptor-tp6-3.ini index 127fcb973c31..4ece43e7bf05 100644 --- a/testing/raptor/raptor/tests/raptor-tp6-3.ini +++ b/testing/raptor/raptor/tests/raptor-tp6-3.ini @@ -54,3 +54,22 @@ apps = chrome test_url = http://fandom.wikia.com/articles/fallout-76-will-live-and-die-on-the-creativity-of-its-playerbase playback_recordings = wikia.mp measure = fcp, loadtime + +[raptor-tp6-imdb-chromium] +apps = chromium +test_url = https://www.imdb.com/title/tt0084967/?ref_=nv_sr_2 +playback_recordings = imdb.mp +measure = fcp, loadtime + +[raptor-tp6-imgur-chromium] +apps = chromium +test_url = https://imgur.com/gallery/m5tYJL6 +playback_recordings = imgur.mp +measure = fcp, loadtime +disabled = Bug 1529918 - raptor-tp6-imgur-chrome fails to load with latest Chromium update + +[raptor-tp6-wikia-chromium] +apps = chromium +test_url = http://fandom.wikia.com/articles/fallout-76-will-live-and-die-on-the-creativity-of-its-playerbase +playback_recordings = wikia.mp +measure = fcp, loadtime diff --git a/testing/raptor/raptor/tests/raptor-tp6-4.ini b/testing/raptor/raptor/tests/raptor-tp6-4.ini index 98030001cd77..92b22c0ff70b 100644 --- a/testing/raptor/raptor/tests/raptor-tp6-4.ini +++ b/testing/raptor/raptor/tests/raptor-tp6-4.ini @@ -41,3 +41,15 @@ apps = chrome test_url = https://yandex.ru/search/?text=barack%20obama&lr=10115 playback_recordings = yandex.mp measure = fcp, loadtime + +[raptor-tp6-bing-chromium] +apps = chromium +test_url = https://www.bing.com/search?q=barack+obama +playback_recordings = bing.mp +measure = fcp, loadtime + +[raptor-tp6-yandex-chromium] +apps = chromium +test_url = https://yandex.ru/search/?text=barack%20obama&lr=10115 +playback_recordings = yandex.mp +measure = fcp, loadtime diff --git a/testing/raptor/raptor/tests/raptor-tp6-5.ini b/testing/raptor/raptor/tests/raptor-tp6-5.ini index c6701b72debe..e729cad52ec6 100644 --- a/testing/raptor/raptor/tests/raptor-tp6-5.ini +++ b/testing/raptor/raptor/tests/raptor-tp6-5.ini @@ -42,3 +42,15 @@ apps = chrome test_url = https://www.microsoft.com/en-us/windows/get-windows-10 playback_recordings = microsoft.mp measure = fcp, loadtime + +[raptor-tp6-apple-chromium] +apps = chromium +test_url = https://www.apple.com/macbook-pro/ +playback_recordings = apple.mp +measure = fcp, loadtime + +[raptor-tp6-microsoft-chromium] +apps = chromium +test_url = https://www.microsoft.com/en-us/windows/get-windows-10 +playback_recordings = microsoft.mp +measure = fcp, loadtime diff --git a/testing/raptor/raptor/tests/raptor-tp6-6.ini b/testing/raptor/raptor/tests/raptor-tp6-6.ini index acf642b91d8c..1467aa786003 100644 --- a/testing/raptor/raptor/tests/raptor-tp6-6.ini +++ b/testing/raptor/raptor/tests/raptor-tp6-6.ini @@ -43,3 +43,16 @@ test_url = https://www.yahoo.com/lifestyle/police-respond-noise-complaint-end-pl playback_pageset_manifest = mitmproxy-recordings-raptor-yahoo-news.manifest playback_recordings = yahoo-news.mp measure = fcp, loadtime + +[raptor-tp6-reddit-chromium] +apps = chromium +test_url = https://www.reddit.com/r/technology/comments/9sqwyh/we_posed_as_100_senators_to_run_ads_on_facebook/ +playback_recordings = reddit.mp +measure = fcp, loadtime + +[raptor-tp6-yahoo-news-chromium] +apps = chromium +test_url = https://www.yahoo.com/lifestyle/police-respond-noise-complaint-end-playing-video-games-respectful-tenants-002329963.html +playback_pageset_manifest = mitmproxy-recordings-raptor-yahoo-news.manifest +playback_recordings = yahoo-news.mp +measure = fcp, loadtime diff --git a/testing/raptor/raptor/tests/raptor-tp6-7.ini b/testing/raptor/raptor/tests/raptor-tp6-7.ini index 4fc67fe96237..bd3c09cc2d0e 100644 --- a/testing/raptor/raptor/tests/raptor-tp6-7.ini +++ b/testing/raptor/raptor/tests/raptor-tp6-7.ini @@ -57,3 +57,23 @@ test_url = https://mail.yahoo.com/ playback_pageset_manifest = mitmproxy-tp6-yahoo-mail.manifest playback_recordings = yahoo-mail.mp measure = fcp, loadtime + +[raptor-tp6-instagram-chromium] +apps = chromium +test_url = https://www.instagram.com/ +playback_recordings = instagram.mp +measure = fcp, loadtime + +[raptor-tp6-twitter-chromium] +apps = chromium +test_url = https://twitter.com/BarackObama +playback_pageset_manifest = mitmproxy-recordings-raptor-twitter.manifest +playback_recordings = twitter.mp +measure = fcp, loadtime + +[raptor-tp6-yahoo-mail-chromium] +apps = chromium +test_url = https://mail.yahoo.com/ +playback_pageset_manifest = mitmproxy-tp6-yahoo-mail.manifest +playback_recordings = yahoo-mail.mp +measure = fcp, loadtime diff --git a/testing/raptor/raptor/tests/raptor-tp6-8-404.ini b/testing/raptor/raptor/tests/raptor-tp6-8-404.ini index 09f20dac609f..e2469b0cd0ef 100644 --- a/testing/raptor/raptor/tests/raptor-tp6-8-404.ini +++ b/testing/raptor/raptor/tests/raptor-tp6-8-404.ini @@ -73,3 +73,31 @@ test_url = https://en.wikipedia.org/wiki/Barack_Obama playback_pageset_manifest = mitmproxy-recordings-raptor-wikipedia-404.manifest playback_recordings = wikipedia.mp measure = fcp, loadtime + +[raptor-tp6-ebay-mitm-404-recordings-202-chromium] +apps = chromium +test_url = https://www.ebay.com/ +playback_pageset_manifest = mitmproxy-recordings-raptor-ebay.manifest +playback_recordings = ebay.mp +measure = fcp, loadtime + +[raptor-tp6-wikipedia-mitm-404-recordings-202-chromium] +apps = chromium +test_url = https://en.wikipedia.org/wiki/Barack_Obama +playback_pageset_manifest = mitmproxy-recordings-raptor-wikipedia.manifest +playback_recordings = wikipedia.mp +measure = fcp, loadtime + +[raptor-tp6-ebay-mitm-404-recordings-404-chromium] +apps = chromium +test_url = https://www.ebay.com/ +playback_pageset_manifest = mitmproxy-recordings-raptor-ebay-404.manifest +playback_recordings = ebay.mp +measure = fcp, loadtime + +[raptor-tp6-wikipedia-mitm-404-recordings-404-chromium] +apps = chromium +test_url = https://en.wikipedia.org/wiki/Barack_Obama +playback_pageset_manifest = mitmproxy-recordings-raptor-wikipedia-404.manifest +playback_recordings = wikipedia.mp +measure = fcp, loadtime diff --git a/testing/raptor/raptor/tests/raptor-tp6-8.ini b/testing/raptor/raptor/tests/raptor-tp6-8.ini index c154e5fd794b..2eabebb2a017 100644 --- a/testing/raptor/raptor/tests/raptor-tp6-8.ini +++ b/testing/raptor/raptor/tests/raptor-tp6-8.ini @@ -44,3 +44,17 @@ test_url = https://en.wikipedia.org/wiki/Barack_Obama playback_pageset_manifest = mitmproxy-recordings-raptor-wikipedia.manifest playback_recordings = wikipedia.mp measure = fcp, loadtime + +[raptor-tp6-ebay-chromium] +apps = chromium +test_url = https://www.ebay.com/ +playback_pageset_manifest = mitmproxy-recordings-raptor-ebay.manifest +playback_recordings = ebay.mp +measure = fcp, loadtime + +[raptor-tp6-wikipedia-chromium] +apps = chromium +test_url = https://en.wikipedia.org/wiki/Barack_Obama +playback_pageset_manifest = mitmproxy-recordings-raptor-wikipedia.manifest +playback_recordings = wikipedia.mp +measure = fcp, loadtime diff --git a/testing/raptor/raptor/tests/raptor-tp6-9.ini b/testing/raptor/raptor/tests/raptor-tp6-9.ini index 6e712be39b00..74e810dea9e9 100644 --- a/testing/raptor/raptor/tests/raptor-tp6-9.ini +++ b/testing/raptor/raptor/tests/raptor-tp6-9.ini @@ -44,3 +44,17 @@ test_url = https://pinterest.com/ playback_pageset_manifest = mitmproxy-recordings-raptor-pinterest.manifest playback_recordings = pinterest.mp measure = fcp, loadtime + +[raptor-tp6-google-mail-chromium] +apps = chromium +test_url = https://mail.google.com/ +playback_pageset_manifest = mitmproxy-recordings-raptor-google-mail.manifest +playback_recordings = google-mail.mp +measure = fcp, loadtime + +[raptor-tp6-pinterest-chromium] +apps = chromium +test_url = https://pinterest.com/ +playback_pageset_manifest = mitmproxy-recordings-raptor-pinterest.manifest +playback_recordings = pinterest.mp +measure = fcp, loadtime diff --git a/testing/raptor/raptor/tests/raptor-unity-webgl.ini b/testing/raptor/raptor/tests/raptor-unity-webgl.ini index cdac6d40774f..45c12766d9a6 100644 --- a/testing/raptor/raptor/tests/raptor-unity-webgl.ini +++ b/testing/raptor/raptor/tests/raptor-unity-webgl.ini @@ -15,6 +15,9 @@ apps = firefox [raptor-unity-webgl-chrome] apps = chrome +[raptor-unity-webgl-chromium] +apps = chromium + [raptor-unity-webgl-geckoview] page_timeout = 900000 # 15 min page_cycles = 1 diff --git a/testing/raptor/raptor/tests/raptor-wasm-godot-baseline.ini b/testing/raptor/raptor/tests/raptor-wasm-godot-baseline.ini index 43ca1272e5f3..7fd283f604b2 100644 --- a/testing/raptor/raptor/tests/raptor-wasm-godot-baseline.ini +++ b/testing/raptor/raptor/tests/raptor-wasm-godot-baseline.ini @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -# Wasm-godot benchmark for firefox and chrome +# Wasm-godot benchmark for firefox and chromium distributions [DEFAULT] type = benchmark diff --git a/testing/raptor/raptor/tests/raptor-wasm-godot-cranelift.ini b/testing/raptor/raptor/tests/raptor-wasm-godot-cranelift.ini index 390fa00de3dc..0092553f0a74 100644 --- a/testing/raptor/raptor/tests/raptor-wasm-godot-cranelift.ini +++ b/testing/raptor/raptor/tests/raptor-wasm-godot-cranelift.ini @@ -2,19 +2,19 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -# Wasm-Godot benchmark for firefox and chrome +# Wasm-Godot benchmark for firefox and chromium distributions [DEFAULT] type = benchmark -test_url = http://localhost:/wasm-godot/index.html +test_url = http://localhost:/wasm-godot/index.html page_cycles = 5 page_timeout = 120000 unit = ms lower_is_better = true alert_threshold = 2.0 -newtab_per_cycle = true -gecko_profile_interval = 1 -gecko_profile_entries = 8000000 +newtab_per_cycle = true +gecko_profile_interval = 1 +gecko_profile_entries = 8000000 [raptor-wasm-godot-cranelift-firefox] apps = firefox diff --git a/testing/raptor/raptor/tests/raptor-wasm-godot-ion.ini b/testing/raptor/raptor/tests/raptor-wasm-godot-ion.ini index 09932e5ff8a3..3135d5c8a761 100644 --- a/testing/raptor/raptor/tests/raptor-wasm-godot-ion.ini +++ b/testing/raptor/raptor/tests/raptor-wasm-godot-ion.ini @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -# Wasm-godot benchmark for firefox and chrome +# Wasm-godot benchmark for firefox and chromium distributions [DEFAULT] type = benchmark diff --git a/testing/raptor/raptor/tests/raptor-wasm-godot.ini b/testing/raptor/raptor/tests/raptor-wasm-godot.ini index b42d0ad7d542..4f051f364eea 100644 --- a/testing/raptor/raptor/tests/raptor-wasm-godot.ini +++ b/testing/raptor/raptor/tests/raptor-wasm-godot.ini @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -# Wasm-godot benchmark for firefox and chrome +# Wasm-godot benchmark for firefox and chromium distributions [DEFAULT] type = benchmark @@ -21,3 +21,6 @@ apps = firefox [raptor-wasm-godot-chrome] apps = chrome + +[raptor-wasm-godot-chromium] +apps = chromium diff --git a/testing/raptor/raptor/tests/raptor-wasm-misc-baseline.ini b/testing/raptor/raptor/tests/raptor-wasm-misc-baseline.ini index 750ecc9f2337..e60e9191cf93 100644 --- a/testing/raptor/raptor/tests/raptor-wasm-misc-baseline.ini +++ b/testing/raptor/raptor/tests/raptor-wasm-misc-baseline.ini @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -# Wasm-Misc benchmark for firefox and chrome +# Wasm-Misc benchmark for firefox and chromium distributions [DEFAULT] type = benchmark diff --git a/testing/raptor/raptor/tests/raptor-wasm-misc-cranelift.ini b/testing/raptor/raptor/tests/raptor-wasm-misc-cranelift.ini index d9c6df6c1b0e..40fcd9e13267 100644 --- a/testing/raptor/raptor/tests/raptor-wasm-misc-cranelift.ini +++ b/testing/raptor/raptor/tests/raptor-wasm-misc-cranelift.ini @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -# Wasm-Misc benchmark for firefox and chrome +# Wasm-Misc benchmark for firefox and chromium distributions [DEFAULT] type = benchmark diff --git a/testing/raptor/raptor/tests/raptor-wasm-misc-ion.ini b/testing/raptor/raptor/tests/raptor-wasm-misc-ion.ini index a4b78a7c52c7..7104f0607c7e 100644 --- a/testing/raptor/raptor/tests/raptor-wasm-misc-ion.ini +++ b/testing/raptor/raptor/tests/raptor-wasm-misc-ion.ini @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -# Wasm-Misc benchmark for firefox and chrome +# Wasm-Misc benchmark for firefox and chromium distributions [DEFAULT] type = benchmark diff --git a/testing/raptor/raptor/tests/raptor-wasm-misc.ini b/testing/raptor/raptor/tests/raptor-wasm-misc.ini index 6da785f7811a..16478a6acb4d 100644 --- a/testing/raptor/raptor/tests/raptor-wasm-misc.ini +++ b/testing/raptor/raptor/tests/raptor-wasm-misc.ini @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -# Wasm-Misc benchmark for firefox and chrome +# Wasm-Misc benchmark for firefox and chromium distributions [DEFAULT] type = benchmark @@ -20,3 +20,6 @@ apps = firefox [raptor-wasm-misc-chrome] apps = chrome + +[raptor-wasm-misc-chromium] +apps = chromium diff --git a/testing/raptor/raptor/tests/raptor-webaudio.ini b/testing/raptor/raptor/tests/raptor-webaudio.ini index b775f8b8ae1c..700e4cc5208a 100644 --- a/testing/raptor/raptor/tests/raptor-webaudio.ini +++ b/testing/raptor/raptor/tests/raptor-webaudio.ini @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -# speedometer benchmark for firefox and chrome +# speedometer benchmark for firefox and chromium distributions [DEFAULT] type = benchmark @@ -20,3 +20,6 @@ apps = firefox [raptor-webaudio-chrome] apps = chrome + +[raptor-webaudio-chromium] +apps = chromium diff --git a/testing/raptor/test/test_manifest.py b/testing/raptor/test/test_manifest.py index 234b2522fa82..5111ee328777 100644 --- a/testing/raptor/test/test_manifest.py +++ b/testing/raptor/test/test_manifest.py @@ -46,6 +46,15 @@ VALID_MANIFESTS = [{'apps': 'firefox', 'lower_is_better': False, 'alert_threshold': 2.0, 'manifest': 'valid_details_1'}, + {'apps': 'chromium', + 'type': 'benchmark', + 'page_cycles': 5, + 'test_url': 'http://www.test-url/goes/here', + 'measure': 'fcp', + 'unit': 'score', + 'lower_is_better': False, + 'alert_threshold': 2.0, + 'manifest': 'valid_details_1'}, {'apps': 'geckoview', 'type': 'pageload', 'cold': True, @@ -80,6 +89,16 @@ INVALID_MANIFESTS = [{'apps': 'firefox', 'alert_threshold': 2.0, 'playback': 'mitmproxy', 'manifest': 'invalid_details_1'}, + {'apps': 'chromium', + 'type': 'pageload', + 'page_cycles': 25, + 'test_url': 'http://www.test-url/goes/here', + 'measure': 'fnbpaint, fcp', + 'unit': 'ms', + 'lower_is_better': True, + 'alert_threshold': 2.0, + 'playback': 'mitmproxy', + 'manifest': 'invalid_details_1'}, {'apps': 'firefox', 'type': 'pageload', 'page_cycles': 25, @@ -96,7 +115,7 @@ INVALID_MANIFESTS = [{'apps': 'firefox', 'manifest': 'invalid_details_2'}] -@pytest.mark.parametrize('app', ['firefox', 'chrome', 'geckoview', 'refbrow', 'fenix']) +@pytest.mark.parametrize('app', ['firefox', 'chrome', 'chromium', 'geckoview', 'refbrow', 'fenix']) def test_get_browser_test_list(app): test_list = get_browser_test_list(app, run_local=True) if app != "fenix": diff --git a/testing/raptor/test/test_raptor.py b/testing/raptor/test/test_raptor.py index 097ab1d832f9..12e8c731623e 100644 --- a/testing/raptor/test/test_raptor.py +++ b/testing/raptor/test/test_raptor.py @@ -42,6 +42,7 @@ class TestBrowserThread(threading.Thread): @pytest.mark.parametrize("raptor_class, app_name", [ [RaptorDesktopFirefox, "firefox"], [RaptorDesktopChrome, "chrome"], + [RaptorDesktopChrome, "chromium"], [RaptorAndroid, "fennec"], [RaptorAndroid, "geckoview"], ]) @@ -93,6 +94,7 @@ def test_start_and_stop_server(raptor): @pytest.mark.parametrize('app', [ 'firefox', pytest.mark.xfail('chrome'), + pytest.mark.xfail('chromium'), ]) def test_start_browser(get_binary, app): binary = get_binary(app)