Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-07 09:54:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1670005 - Remove all references to Fennec from performance testing r=perftest-reviewers,sparky

Browse Source 
Differential Revision: https://phabricator.services.mozilla.com/D109486
This commit is contained in:
Dave Hunt 2021-03-24 10:44:22 +00:00
parent bb7ce0f853
commit f7f9d0978a
29 changed files with 119 additions and 310 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
taskcluster/ci/condprof/kind.yml
View File

@ -193,40 +193,6 @@ jobs:
- name: public/condprof - name: public/condprof
path: archive path: archive
type: directory type: directory
android-hw-g5-7-0-arm7-api-16-fennec68:
worker-type: t-bitbar-gw-perf-g5
description: Creates or update conditioned profiles on Fennec68+G5
treeherder:
symbol: condprof(fennec68)
platform: android-hw-g5-7-0-arm7-api-16/opt
index:
product: firefox
job-name: condprof-g5-firefox
dependencies:
build: build-linux64-shippable/opt
fetches:
build:
- artifact: target.condprof.tests.tar.gz
toolchain:
- linux64-geckodriver
run:
run-as-root: true
command: >-
curl -L -o target.apk https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-esr68.signed-nightly.nightly.latest.mobile.android-api-16-release-opt/artifacts/public/build/target.apk &&
adb install -r target.apk &&
python3 virtualenv/virtualenv.py . &&
bin/python3 condprof/main.py
--force-new
--geckodriver ${MOZ_FETCHES_DIR}/geckodriver
--device-name g5
--firefox org.mozilla.firefox
--scenario settled
../../archive
worker:
artifacts:
- name: public/condprof
path: archive
type: directory
android-hw-p2-8-0-aarch64-fenix: android-hw-p2-8-0-aarch64-fenix:
worker-type: t-bitbar-gw-perf-p2 worker-type: t-bitbar-gw-perf-p2
description: Creates or update conditioned profiles on Fenix+P2 description: Creates or update conditioned profiles on Fenix+P2
@ -297,38 +263,3 @@ jobs:
- name: public/condprof - name: public/condprof
path: archive path: archive
type: directory type: directory
android-hw-p2-8-0-aarch64-fennec68:
worker-type: t-bitbar-gw-perf-p2
description: Creates or update conditioned profiles on Fennec68+P2
treeherder:
symbol: condprof(fennec68)
platform: android-hw-p2-8-0-android-aarch64/opt
index:
product: firefox
job-name: condprof-p2_aarch64-firefox
dependencies:
build: build-linux64-shippable/opt
fetches:
build:
- artifact: target.condprof.tests.tar.gz
toolchain:
- linux64-geckodriver
run:
run-as-root: true
command: >-
curl -L -o target.apk https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-esr68.signed-nightly.nightly.latest.mobile.android-aarch64-release-opt/artifacts/public/build/target.apk &&
adb install -r target.apk &&
rm -rf target.apk &&
python3 virtualenv/virtualenv.py . &&
bin/python3 condprof/main.py
--force-new
--geckodriver ${MOZ_FETCHES_DIR}/geckodriver
--device-name p2_aarch64
--firefox org.mozilla.firefox
--scenario settled
../../archive
worker:
artifacts:
- name: public/condprof
path: archive
type: directory

2
taskcluster/ci/config.yml
View File

@ -39,8 +39,6 @@ treeherder:
'Btime-P-refbrow-wr': 'Browsertime power tests on reference browser with webrender enabled' 'Btime-P-refbrow-wr': 'Browsertime power tests on reference browser with webrender enabled'
'Btime-fenix': 'Browsertime performance tests on Fenix' 'Btime-fenix': 'Browsertime performance tests on Fenix'
'Btime-fenix-wr': 'Browsertime performance tests on Fenix with webrender enabled' 'Btime-fenix-wr': 'Browsertime performance tests on Fenix with webrender enabled'
'Btime-fennec': 'Browsertime performance tests on Fennec'
'Btime-fennec-1proc': 'Browsertime performance tests on Fennec without e10s'
'Btime-refbrow': 'Browsertime performance tests on the reference browser' 'Btime-refbrow': 'Browsertime performance tests on the reference browser'
'Btime-refbrow-wr': 'Browsertime performance tests on the reference browser with webrender enabled' 'Btime-refbrow-wr': 'Browsertime performance tests on the reference browser with webrender enabled'
'Btime-wr': 'Browsertime performance tests on Android with webrender enabled' 'Btime-wr': 'Browsertime performance tests on Android with webrender enabled'

20
taskcluster/ci/test/browsertime-mobile.yml
View File

@ -8,7 +8,6 @@ job-defaults:
run-on-projects: run-on-projects:
by-app: by-app:
refbrow: [] refbrow: []
fennec: []
default: default:
by-test-platform: by-test-platform:
android-hw-p2-.*aarch64.*/pgo: ["mozilla-central"] android-hw-p2-.*aarch64.*/pgo: ["mozilla-central"]
@ -27,14 +26,6 @@ job-defaults:
default: default:
index: mobile.v2.fenix.nightly.latest.armeabi-v7a index: mobile.v2.fenix.nightly.latest.armeabi-v7a
name: build/armeabi-v7a/target.apk name: build/armeabi-v7a/target.apk
fennec:
by-test-platform:
android-hw-p2-8-0-android-aarch64.*:
index: gecko.v2.mozilla-esr68.signed-nightly.nightly.latest.mobile.android-aarch64-release-opt
name: build/target.apk
default:
index: gecko.v2.mozilla-esr68.signed-nightly.nightly.latest.mobile.android-api-16-release-opt
name: build/target.apk
refbrow: refbrow:
by-test-platform: by-test-platform:
android-hw.*-aarch64.*/.*: android-hw.*-aarch64.*/.*:
@ -50,14 +41,12 @@ job-defaults:
by-app: by-app:
fenix: org.mozilla.fenix.IntentReceiverActivity fenix: org.mozilla.fenix.IntentReceiverActivity
geckoview: org.mozilla.geckoview_example.GeckoViewActivity geckoview: org.mozilla.geckoview_example.GeckoViewActivity
fennec: org.mozilla.gecko.BrowserApp
refbrow: org.mozilla.reference.browser.BrowserTestActivity refbrow: org.mozilla.reference.browser.BrowserTestActivity
default: None default: None
binary-path: binary-path:
by-app: by-app:
fenix: org.mozilla.fenix fenix: org.mozilla.fenix
geckoview: org.mozilla.geckoview_example geckoview: org.mozilla.geckoview_example
fennec: org.mozilla.firefox
refbrow: org.mozilla.reference.browser.raptor refbrow: org.mozilla.reference.browser.raptor
default: None default: None
mozharness: mozharness:
@ -74,7 +63,6 @@ job-defaults:
variants: variants:
by-app: by-app:
chrome-m: [] chrome-m: []
fennec: []
default: ["webrender"] default: ["webrender"]
python-3: true python-3: true
@ -114,7 +102,7 @@ browsertime-tp6m:
- wikipedia - wikipedia
- youtube - youtube
- [youtube-watch, youtube-w] - [youtube-watch, youtube-w]
apps: ["geckoview", "fenix", "fennec", "chrome-m", "refbrow"] apps: ["geckoview", "fenix", "chrome-m", "refbrow"]
tier: tier:
by-raptor-test: by-raptor-test:
amazon: amazon:
@ -122,13 +110,11 @@ browsertime-tp6m:
fenix: 1 fenix: 1
geckoview: 1 geckoview: 1
refbrow: 2 refbrow: 2
fennec: 2
default: 3 default: 3
default: default:
by-app: by-app:
geckoview: 1 geckoview: 1
refbrow: 2 refbrow: 2
fennec: 2
default: 3 default: 3
webrender-run-on-projects: webrender-run-on-projects:
by-raptor-test: by-raptor-test:
@ -136,7 +122,6 @@ browsertime-tp6m:
by-app: by-app:
fenix: [] fenix: []
chrome-m: [] chrome-m: []
fennec: []
geckoview: geckoview:
&sheriffed-platforms &sheriffed-platforms
by-test-platform: by-test-platform:
@ -152,7 +137,6 @@ browsertime-tp6m:
fenix: [] fenix: []
chrome-m: [] chrome-m: []
refbrow: [] refbrow: []
fennec: []
geckoview: *sheriffed-platforms geckoview: *sheriffed-platforms
default: default:
by-test-platform: by-test-platform:
@ -314,7 +298,7 @@ browsertime-tp6m-profiling:
browsertime-speedometer-mobile: browsertime-speedometer-mobile:
description: "Browsertime Speedometer on Android" description: "Browsertime Speedometer on Android"
apps: ["geckoview", "fenix", "fennec", "chrome-m"] apps: ["geckoview", "fenix", "chrome-m"]
tier: tier:
by-app: by-app:
fenix: 1 fenix: 1

6
taskcluster/taskgraph/transforms/raptor.py
View File

@ -246,12 +246,6 @@ def add_extra_options(config, tests):
extra_options.append("--browsertime-video") extra_options.append("--browsertime-video")
test["attributes"]["run-visual-metrics"] = True test["attributes"]["run-visual-metrics"] = True
if test.get("app", "") == "fennec" and test["test-name"].startswith(
"browsertime"
):
# Bug 1645181: Conditioned profiles cause problems
extra_options.append("--no-conditioned-profile")
if "app" in test: if "app" in test:
extra_options.append("--app={}".format(test.pop("app"))) extra_options.append("--app={}".format(test.pop("app")))

32
testing/condprofile/condprof/android.py
View File

@ -22,13 +22,10 @@ from condprof.util import write_yml_file, logger, DEFAULT_PREFS, BaseEnv
class AndroidDevice: class AndroidDevice:
def __init__(self, app_name, marionette_port=2828, verbose=False): def __init__(self, app_name, marionette_port=2828, verbose=False):
self.app_name = app_name self.app_name = app_name
self.fennec = "firefox" in app_name
# XXX make that an option # XXX make that an option
if "fenix" in app_name: if "fenix" in app_name:
self.activity = "org.mozilla.fenix.IntentReceiverActivity" self.activity = "org.mozilla.fenix.IntentReceiverActivity"
elif self.fennec:
self.activity = None
else: else:
self.activity = "org.mozilla.geckoview_example.GeckoViewActivity" self.activity = "org.mozilla.geckoview_example.GeckoViewActivity"
self.verbose = verbose self.verbose = verbose
@ -141,32 +138,9 @@ class AndroidDevice:
# an on-device config.yml file # an on-device config.yml file
intent = "android.intent.action.VIEW" intent = "android.intent.action.VIEW"
device.stop_application(self.app_name) device.stop_application(self.app_name)
if self.fennec: device.launch_application(
# XXX does the Fennec app picks up the YML file ? self.app_name, self.activity, intent, extras=None, url="about:blank"
extra_args = [ )
"-profile",
self.remote_profile,
"--es",
"env0",
"LOG_VERBOSE=1",
"--es",
"env1",
"R_LOG_LEVEL=6",
"--es",
"env2",
"MOZ_WEBRENDER=0",
]
device.launch_fennec(
self.app_name,
extra_args=extra_args,
url="about:blank",
fail_if_running=False,
)
else:
device.launch_application(
self.app_name, self.activity, intent, extras=None, url="about:blank"
)
if not device.process_exist(self.app_name): if not device.process_exist(self.app_name):
raise Exception("Could not start %s" % self.app_name) raise Exception("Could not start %s" % self.app_name)

110
testing/perfdocs/generated/raptor.rst
View File

@ -711,13 +711,13 @@ A set of test pages that are run as live sites instead of recorded versions. The
Mobile Mobile
------ ------
Page-load performance test suite on Android. The links direct to the actual websites that are being tested. (WX: WebExtension, BT: Browsertime, GV: Geckoview, RB: Refbrow, FE: Fenix, F68: Fennec68, CH-M: Chrome mobile) Page-load performance test suite on Android. The links direct to the actual websites that are being tested. (WX: WebExtension, BT: Browsertime, GV: Geckoview, RB: Refbrow, FE: Fenix, CH-M: Chrome mobile)
.. dropdown:: allrecipes (BT, GV, FE, RB, F68, CH-M) .. dropdown:: allrecipes (BT, GV, FE, RB, CH-M)
* **alert on**: fcp, loadtime * **alert on**: fcp, loadtime
* **alert threshold**: 2.0 * **alert threshold**: 2.0
* **apps**: geckoview, fenix, refbrow, fennec, chrome-m * **apps**: geckoview, fenix, refbrow, chrome-m
* **browser cycles**: 15 * **browser cycles**: 15
* **expected**: pass * **expected**: pass
* **lower is better**: true * **lower is better**: true
@ -732,11 +732,11 @@ Page-load performance test suite on Android. The links direct to the actual webs
* **use live sites**: false * **use live sites**: false
.. dropdown:: amazon (BT, GV, FE, RB, F68, CH-M) .. dropdown:: amazon (BT, GV, FE, RB, CH-M)
* **alert on**: fcp, loadtime * **alert on**: fcp, loadtime
* **alert threshold**: 2.0 * **alert threshold**: 2.0
* **apps**: geckoview, fenix, refbrow, fennec, chrome-m * **apps**: geckoview, fenix, refbrow, chrome-m
* **browser cycles**: 15 * **browser cycles**: 15
* **expected**: pass * **expected**: pass
* **lower is better**: true * **lower is better**: true
@ -751,11 +751,11 @@ Page-load performance test suite on Android. The links direct to the actual webs
* **use live sites**: false * **use live sites**: false
.. dropdown:: amazon-search (BT, GV, FE, RB, F68, CH-M) .. dropdown:: amazon-search (BT, GV, FE, RB, CH-M)
* **alert on**: fcp, loadtime * **alert on**: fcp, loadtime
* **alert threshold**: 2.0 * **alert threshold**: 2.0
* **apps**: geckoview, fenix, refbrow, fennec, chrome-m * **apps**: geckoview, fenix, refbrow, chrome-m
* **browser cycles**: 15 * **browser cycles**: 15
* **expected**: pass * **expected**: pass
* **lower is better**: true * **lower is better**: true
@ -770,11 +770,11 @@ Page-load performance test suite on Android. The links direct to the actual webs
* **use live sites**: false * **use live sites**: false
.. dropdown:: bbc (BT, GV, FE, RB, F68, CH-M) .. dropdown:: bbc (BT, GV, FE, RB, CH-M)
* **alert on**: fcp, loadtime * **alert on**: fcp, loadtime
* **alert threshold**: 2.0 * **alert threshold**: 2.0
* **apps**: geckoview, fenix, refbrow, fennec, chrome-m * **apps**: geckoview, fenix, refbrow, chrome-m
* **browser cycles**: 15 * **browser cycles**: 15
* **expected**: pass * **expected**: pass
* **lower is better**: true * **lower is better**: true
@ -789,11 +789,11 @@ Page-load performance test suite on Android. The links direct to the actual webs
* **use live sites**: false * **use live sites**: false
.. dropdown:: bing (BT, GV, FE, RB, F68, CH-M) .. dropdown:: bing (BT, GV, FE, RB, CH-M)
* **alert on**: fcp, loadtime * **alert on**: fcp, loadtime
* **alert threshold**: 2.0 * **alert threshold**: 2.0
* **apps**: geckoview, fenix, refbrow, fennec, chrome-m * **apps**: geckoview, fenix, refbrow, chrome-m
* **browser cycles**: 15 * **browser cycles**: 15
* **expected**: pass * **expected**: pass
* **lower is better**: true * **lower is better**: true
@ -808,11 +808,11 @@ Page-load performance test suite on Android. The links direct to the actual webs
* **use live sites**: false * **use live sites**: false
.. dropdown:: bing-search-restaurants (BT, GV, FE, RB, F68, CH-M) .. dropdown:: bing-search-restaurants (BT, GV, FE, RB, CH-M)
* **alert on**: fcp, loadtime * **alert on**: fcp, loadtime
* **alert threshold**: 2.0 * **alert threshold**: 2.0
* **apps**: geckoview, fenix, refbrow, fennec, chrome-m * **apps**: geckoview, fenix, refbrow, chrome-m
* **browser cycles**: 15 * **browser cycles**: 15
* **expected**: pass * **expected**: pass
* **lower is better**: true * **lower is better**: true
@ -827,11 +827,11 @@ Page-load performance test suite on Android. The links direct to the actual webs
* **use live sites**: false * **use live sites**: false
.. dropdown:: booking (BT, GV, FE, RB, F68, CH-M) .. dropdown:: booking (BT, GV, FE, RB, CH-M)
* **alert on**: fcp, loadtime * **alert on**: fcp, loadtime
* **alert threshold**: 2.0 * **alert threshold**: 2.0
* **apps**: geckoview, fenix, refbrow, fennec, chrome-m * **apps**: geckoview, fenix, refbrow, chrome-m
* **browser cycles**: 15 * **browser cycles**: 15
* **expected**: pass * **expected**: pass
* **lower is better**: true * **lower is better**: true
@ -846,11 +846,11 @@ Page-load performance test suite on Android. The links direct to the actual webs
* **use live sites**: false * **use live sites**: false
.. dropdown:: cnn (BT, GV, FE, RB, F68, CH-M) .. dropdown:: cnn (BT, GV, FE, RB, CH-M)
* **alert on**: fcp, loadtime * **alert on**: fcp, loadtime
* **alert threshold**: 2.0 * **alert threshold**: 2.0
* **apps**: geckoview, fenix, refbrow, fennec, chrome-m * **apps**: geckoview, fenix, refbrow, chrome-m
* **browser cycles**: 15 * **browser cycles**: 15
* **expected**: pass * **expected**: pass
* **lower is better**: true * **lower is better**: true
@ -865,11 +865,11 @@ Page-load performance test suite on Android. The links direct to the actual webs
* **use live sites**: false * **use live sites**: false
.. dropdown:: cnn-ampstories (BT, GV, FE, RB, F68, CH-M) .. dropdown:: cnn-ampstories (BT, GV, FE, RB, CH-M)
* **alert on**: fcp, loadtime * **alert on**: fcp, loadtime
* **alert threshold**: 2.0 * **alert threshold**: 2.0
* **apps**: geckoview, fenix, refbrow, fennec, chrome-m * **apps**: geckoview, fenix, refbrow, chrome-m
* **browser cycles**: 15 * **browser cycles**: 15
* **expected**: pass * **expected**: pass
* **lower is better**: true * **lower is better**: true
@ -884,11 +884,11 @@ Page-load performance test suite on Android. The links direct to the actual webs
* **use live sites**: false * **use live sites**: false
.. dropdown:: ebay-kleinanzeigen (BT, GV, FE, RB, F68, CH-M) .. dropdown:: ebay-kleinanzeigen (BT, GV, FE, RB, CH-M)
* **alert on**: fcp, loadtime * **alert on**: fcp, loadtime
* **alert threshold**: 2.0 * **alert threshold**: 2.0
* **apps**: geckoview, fenix, refbrow, fennec, chrome-m * **apps**: geckoview, fenix, refbrow, chrome-m
* **browser cycles**: 15 * **browser cycles**: 15
* **expected**: pass * **expected**: pass
* **lower is better**: true * **lower is better**: true
@ -903,11 +903,11 @@ Page-load performance test suite on Android. The links direct to the actual webs
* **use live sites**: false * **use live sites**: false
.. dropdown:: ebay-kleinanzeigen-search (BT, GV, FE, RB, F68, CH-M) .. dropdown:: ebay-kleinanzeigen-search (BT, GV, FE, RB, CH-M)
* **alert on**: fcp, loadtime * **alert on**: fcp, loadtime
* **alert threshold**: 2.0 * **alert threshold**: 2.0
* **apps**: geckoview, fenix, refbrow, fennec, chrome-m * **apps**: geckoview, fenix, refbrow, chrome-m
* **browser cycles**: 15 * **browser cycles**: 15
* **expected**: pass * **expected**: pass
* **lower is better**: true * **lower is better**: true
@ -922,11 +922,11 @@ Page-load performance test suite on Android. The links direct to the actual webs
* **use live sites**: false * **use live sites**: false
.. dropdown:: espn (BT, GV, FE, RB, F68, CH-M) .. dropdown:: espn (BT, GV, FE, RB, CH-M)
* **alert on**: fcp, loadtime * **alert on**: fcp, loadtime
* **alert threshold**: 2.0 * **alert threshold**: 2.0
* **apps**: geckoview, fenix, refbrow, fennec, chrome-m * **apps**: geckoview, fenix, refbrow, chrome-m
* **browser cycles**: 15 * **browser cycles**: 15
* **expected**: pass * **expected**: pass
* **lower is better**: true * **lower is better**: true
@ -941,11 +941,11 @@ Page-load performance test suite on Android. The links direct to the actual webs
* **use live sites**: false * **use live sites**: false
.. dropdown:: facebook (BT, GV, FE, RB, F68, CH-M) .. dropdown:: facebook (BT, GV, FE, RB, CH-M)
* **alert on**: fcp, loadtime * **alert on**: fcp, loadtime
* **alert threshold**: 2.0 * **alert threshold**: 2.0
* **apps**: geckoview, fenix, refbrow, fennec, chrome-m * **apps**: geckoview, fenix, refbrow, chrome-m
* **browser cycles**: 15 * **browser cycles**: 15
* **expected**: pass * **expected**: pass
* **lower is better**: true * **lower is better**: true
@ -960,11 +960,11 @@ Page-load performance test suite on Android. The links direct to the actual webs
* **use live sites**: false * **use live sites**: false
.. dropdown:: facebook-cristiano (BT, GV, FE, RB, F68, CH-M) .. dropdown:: facebook-cristiano (BT, GV, FE, RB, CH-M)
* **alert on**: fcp, loadtime * **alert on**: fcp, loadtime
* **alert threshold**: 2.0 * **alert threshold**: 2.0
* **apps**: geckoview, fenix, refbrow, fennec, chrome-m * **apps**: geckoview, fenix, refbrow, chrome-m
* **browser cycles**: 15 * **browser cycles**: 15
* **expected**: pass * **expected**: pass
* **lower is better**: true * **lower is better**: true
@ -979,11 +979,11 @@ Page-load performance test suite on Android. The links direct to the actual webs
* **use live sites**: false * **use live sites**: false
.. dropdown:: google (BT, GV, FE, RB, F68, CH-M) .. dropdown:: google (BT, GV, FE, RB, CH-M)
* **alert on**: fcp, loadtime * **alert on**: fcp, loadtime
* **alert threshold**: 2.0 * **alert threshold**: 2.0
* **apps**: geckoview, fenix, refbrow, fennec, chrome-m * **apps**: geckoview, fenix, refbrow, chrome-m
* **browser cycles**: 15 * **browser cycles**: 15
* **expected**: pass * **expected**: pass
* **lower is better**: true * **lower is better**: true
@ -998,11 +998,11 @@ Page-load performance test suite on Android. The links direct to the actual webs
* **use live sites**: false * **use live sites**: false
.. dropdown:: google-maps (BT, GV, FE, RB, F68, CH-M) .. dropdown:: google-maps (BT, GV, FE, RB, CH-M)
* **alert on**: fcp, loadtime * **alert on**: fcp, loadtime
* **alert threshold**: 2.0 * **alert threshold**: 2.0
* **apps**: geckoview, fenix, refbrow, fennec, chrome-m * **apps**: geckoview, fenix, refbrow, chrome-m
* **browser cycles**: 15 * **browser cycles**: 15
* **expected**: pass * **expected**: pass
* **lower is better**: true * **lower is better**: true
@ -1018,11 +1018,11 @@ Page-load performance test suite on Android. The links direct to the actual webs
* **use live sites**: false * **use live sites**: false
.. dropdown:: google-search-restaurants (BT, GV, FE, RB, F68, CH-M) .. dropdown:: google-search-restaurants (BT, GV, FE, RB, CH-M)
* **alert on**: fcp, loadtime * **alert on**: fcp, loadtime
* **alert threshold**: 2.0 * **alert threshold**: 2.0
* **apps**: geckoview, fenix, refbrow, fennec, chrome-m * **apps**: geckoview, fenix, refbrow, chrome-m
* **browser cycles**: 15 * **browser cycles**: 15
* **expected**: pass * **expected**: pass
* **lower is better**: true * **lower is better**: true
@ -1037,11 +1037,11 @@ Page-load performance test suite on Android. The links direct to the actual webs
* **use live sites**: false * **use live sites**: false
.. dropdown:: imdb (BT, GV, FE, RB, F68, CH-M) .. dropdown:: imdb (BT, GV, FE, RB, CH-M)
* **alert on**: fcp, loadtime * **alert on**: fcp, loadtime
* **alert threshold**: 2.0 * **alert threshold**: 2.0
* **apps**: geckoview, fenix, refbrow, fennec, chrome-m * **apps**: geckoview, fenix, refbrow, chrome-m
* **browser cycles**: 15 * **browser cycles**: 15
* **expected**: pass * **expected**: pass
* **lower is better**: true * **lower is better**: true
@ -1056,11 +1056,11 @@ Page-load performance test suite on Android. The links direct to the actual webs
* **use live sites**: false * **use live sites**: false
.. dropdown:: instagram (BT, GV, FE, RB, F68, CH-M) .. dropdown:: instagram (BT, GV, FE, RB, CH-M)
* **alert on**: fcp, loadtime * **alert on**: fcp, loadtime
* **alert threshold**: 2.0 * **alert threshold**: 2.0
* **apps**: geckoview, fenix, refbrow, fennec, chrome-m * **apps**: geckoview, fenix, refbrow, chrome-m
* **browser cycles**: 15 * **browser cycles**: 15
* **expected**: pass * **expected**: pass
* **lower is better**: true * **lower is better**: true
@ -1075,11 +1075,11 @@ Page-load performance test suite on Android. The links direct to the actual webs
* **use live sites**: false * **use live sites**: false
.. dropdown:: jianshu (BT, GV, FE, RB, F68, CH-M) .. dropdown:: jianshu (BT, GV, FE, RB, CH-M)
* **alert on**: fcp, loadtime * **alert on**: fcp, loadtime
* **alert threshold**: 2.0 * **alert threshold**: 2.0
* **apps**: geckoview, fenix, refbrow, fennec, chrome-m * **apps**: geckoview, fenix, refbrow, chrome-m
* **browser cycles**: 15 * **browser cycles**: 15
* **expected**: pass * **expected**: pass
* **lower is better**: true * **lower is better**: true
@ -1094,11 +1094,11 @@ Page-load performance test suite on Android. The links direct to the actual webs
* **use live sites**: false * **use live sites**: false
.. dropdown:: microsoft-support (BT, GV, FE, RB, F68, CH-M) .. dropdown:: microsoft-support (BT, GV, FE, RB, CH-M)
* **alert on**: fcp, loadtime * **alert on**: fcp, loadtime
* **alert threshold**: 2.0 * **alert threshold**: 2.0
* **apps**: geckoview, fenix, refbrow, fennec, chrome-m * **apps**: geckoview, fenix, refbrow, chrome-m
* **browser cycles**: 15 * **browser cycles**: 15
* **expected**: pass * **expected**: pass
* **lower is better**: true * **lower is better**: true
@ -1113,11 +1113,11 @@ Page-load performance test suite on Android. The links direct to the actual webs
* **use live sites**: false * **use live sites**: false
.. dropdown:: reddit (BT, GV, FE, RB, F68, CH-M) .. dropdown:: reddit (BT, GV, FE, RB, CH-M)
* **alert on**: fcp, loadtime * **alert on**: fcp, loadtime
* **alert threshold**: 2.0 * **alert threshold**: 2.0
* **apps**: geckoview, fenix, refbrow, fennec, chrome-m * **apps**: geckoview, fenix, refbrow, chrome-m
* **browser cycles**: 15 * **browser cycles**: 15
* **expected**: pass * **expected**: pass
* **lower is better**: true * **lower is better**: true
@ -1132,11 +1132,11 @@ Page-load performance test suite on Android. The links direct to the actual webs
* **use live sites**: false * **use live sites**: false
.. dropdown:: stackoverflow (BT, GV, FE, RB, F68, CH-M) .. dropdown:: stackoverflow (BT, GV, FE, RB, CH-M)
* **alert on**: fcp, loadtime * **alert on**: fcp, loadtime
* **alert threshold**: 2.0 * **alert threshold**: 2.0
* **apps**: geckoview, fenix, refbrow, fennec, chrome-m * **apps**: geckoview, fenix, refbrow, chrome-m
* **browser cycles**: 15 * **browser cycles**: 15
* **expected**: pass * **expected**: pass
* **lower is better**: true * **lower is better**: true
@ -1151,11 +1151,11 @@ Page-load performance test suite on Android. The links direct to the actual webs
* **use live sites**: false * **use live sites**: false
.. dropdown:: web-de (BT, GV, FE, RB, F68, CH-M) .. dropdown:: web-de (BT, GV, FE, RB, CH-M)
* **alert on**: fcp, loadtime * **alert on**: fcp, loadtime
* **alert threshold**: 2.0 * **alert threshold**: 2.0
* **apps**: geckoview, fenix, refbrow, fennec, chrome-m * **apps**: geckoview, fenix, refbrow, chrome-m
* **browser cycles**: 15 * **browser cycles**: 15
* **expected**: pass * **expected**: pass
* **lower is better**: true * **lower is better**: true
@ -1170,11 +1170,11 @@ Page-load performance test suite on Android. The links direct to the actual webs
* **use live sites**: false * **use live sites**: false
.. dropdown:: wikipedia (BT, GV, FE, RB, F68, CH-M) .. dropdown:: wikipedia (BT, GV, FE, RB, CH-M)
* **alert on**: fcp, loadtime * **alert on**: fcp, loadtime
* **alert threshold**: 2.0 * **alert threshold**: 2.0
* **apps**: geckoview, fenix, refbrow, fennec, chrome-m * **apps**: geckoview, fenix, refbrow, chrome-m
* **browser cycles**: 15 * **browser cycles**: 15
* **expected**: pass * **expected**: pass
* **lower is better**: true * **lower is better**: true
@ -1189,11 +1189,11 @@ Page-load performance test suite on Android. The links direct to the actual webs
* **use live sites**: false * **use live sites**: false
.. dropdown:: youtube (BT, GV, FE, RB, F68, CH-M) .. dropdown:: youtube (BT, GV, FE, RB, CH-M)
* **alert on**: fcp, loadtime * **alert on**: fcp, loadtime
* **alert threshold**: 2.0 * **alert threshold**: 2.0
* **apps**: geckoview, fenix, refbrow, fennec, chrome-m * **apps**: geckoview, fenix, refbrow, chrome-m
* **browser cycles**: 15 * **browser cycles**: 15
* **expected**: pass * **expected**: pass
* **lower is better**: true * **lower is better**: true
@ -1208,11 +1208,11 @@ Page-load performance test suite on Android. The links direct to the actual webs
* **use live sites**: false * **use live sites**: false
.. dropdown:: youtube-watch (BT, GV, FE, RB, F68, CH-M) .. dropdown:: youtube-watch (BT, GV, FE, RB, CH-M)
* **alert on**: fcp, loadtime * **alert on**: fcp, loadtime
* **alert threshold**: 2.0 * **alert threshold**: 2.0
* **apps**: geckoview, fenix, refbrow, fennec, chrome-m * **apps**: geckoview, fenix, refbrow, chrome-m
* **browser cycles**: 15 * **browser cycles**: 15
* **expected**: pass * **expected**: pass
* **lower is better**: true * **lower is better**: true

2
testing/raptor/mach_commands.py
View File

@ -31,7 +31,7 @@ HERE = os.path.dirname(os.path.realpath(__file__))
BENCHMARK_REPOSITORY = "https://github.com/mozilla/perf-automation" BENCHMARK_REPOSITORY = "https://github.com/mozilla/perf-automation"
BENCHMARK_REVISION = "e19a0865c946ae2f9a64dd25614b1c275a3996b2" BENCHMARK_REVISION = "e19a0865c946ae2f9a64dd25614b1c275a3996b2"
ANDROID_BROWSERS = ["fennec", "geckoview", "refbrow", "fenix", "chrome-m"] ANDROID_BROWSERS = ["geckoview", "refbrow", "fenix", "chrome-m"]
class RaptorRunner(MozbuildObject): class RaptorRunner(MozbuildObject):

12
testing/raptor/raptor/cmdline.py
View File

@ -11,8 +11,7 @@ import platform
from mozlog.commandline import add_logging_group from mozlog.commandline import add_logging_group
(FIREFOX, CHROME, CHROMIUM) = DESKTOP_APPS = ["firefox", "chrome", "chromium"] (FIREFOX, CHROME, CHROMIUM) = DESKTOP_APPS = ["firefox", "chrome", "chromium"]
(FENNEC, GECKOVIEW, REFBROW, FENIX, CHROME_ANDROID) = FIREFOX_ANDROID_APPS = [ (GECKOVIEW, REFBROW, FENIX, CHROME_ANDROID) = FIREFOX_ANDROID_APPS = [
"fennec",
"geckoview", "geckoview",
"refbrow", "refbrow",
"fenix", "fenix",
@ -24,7 +23,6 @@ APPS = {
FIREFOX: {"long_name": "Firefox Desktop"}, FIREFOX: {"long_name": "Firefox Desktop"},
CHROME: {"long_name": "Google Chrome Desktop"}, CHROME: {"long_name": "Google Chrome Desktop"},
CHROMIUM: {"long_name": "Google Chromium Desktop"}, CHROMIUM: {"long_name": "Google Chromium Desktop"},
FENNEC: {"long_name": "Firefox Fennec on Android"},
GECKOVIEW: { GECKOVIEW: {
"long_name": "Firefox GeckoView on Android", "long_name": "Firefox GeckoView on Android",
"default_activity": "org.mozilla.geckoview_example.GeckoViewActivity", "default_activity": "org.mozilla.geckoview_example.GeckoViewActivity",
@ -122,7 +120,7 @@ def create_parser(mach_interface=False):
dest="power_test", dest="power_test",
action="store_true", action="store_true",
help="Use Raptor to measure power usage on Android browsers (Geckoview Example, " help="Use Raptor to measure power usage on Android browsers (Geckoview Example, "
"Fenix, Refbrow, and Fennec) as well as on Intel-based MacOS machines that have " "Fenix, and Refbrow) as well as on Intel-based MacOS machines that have "
"Intel Power Gadget installed.", "Intel Power Gadget installed.",
) )
add_arg( add_arg(
@ -476,7 +474,7 @@ def verify_options(parser, args):
parser.error("Gecko profiling is not supported on Chrome/Chromium!") parser.error("Gecko profiling is not supported on Chrome/Chromium!")
if args.power_test: if args.power_test:
if args.app not in ["fennec", "geckoview", "refbrow", "fenix"]: if args.app not in ["geckoview", "refbrow", "fenix"]:
if platform.system().lower() not in ("darwin",): if platform.system().lower() not in ("darwin",):
parser.error( parser.error(
"Power tests are only available on MacOS desktop machines or " "Power tests are only available on MacOS desktop machines or "
@ -485,14 +483,14 @@ def verify_options(parser, args):
) )
if args.cpu_test: if args.cpu_test:
if args.app not in ["fennec", "geckoview", "refbrow", "fenix"]: if args.app not in ["geckoview", "refbrow", "fenix"]:
parser.error( parser.error(
"CPU test is only supported when running Raptor on Firefox Android " "CPU test is only supported when running Raptor on Firefox Android "
"browsers!" "browsers!"
) )
if args.memory_test: if args.memory_test:
if args.app not in ["fennec", "geckoview", "refbrow", "fenix"]: if args.app not in ["geckoview", "refbrow", "fenix"]:
parser.error( parser.error(
"Memory test is only supported when running Raptor on Firefox Android " "Memory test is only supported when running Raptor on Firefox Android "
"browsers!" "browsers!"

56
testing/raptor/raptor/perfdocs/config.yml
View File

@ -40,35 +40,35 @@ suites:
yandex: "BT, FF, CH, CU" yandex: "BT, FF, CH, CU"
youtube: "BT, FF, CH, CU" youtube: "BT, FF, CH, CU"
mobile: mobile:
description: "Page-load performance test suite on Android. The links direct to the actual websites that are being tested. (WX: WebExtension, BT: Browsertime, GV: Geckoview, RB: Refbrow, FE: Fenix, F68: Fennec68, CH-M: Chrome mobile)" description: "Page-load performance test suite on Android. The links direct to the actual websites that are being tested. (WX: WebExtension, BT: Browsertime, GV: Geckoview, RB: Refbrow, FE: Fenix, CH-M: Chrome mobile)"
tests: tests:
amazon: "BT, GV, FE, RB, F68, CH-M" amazon: "BT, GV, FE, RB, CH-M"
youtube: "BT, GV, FE, RB, F68, CH-M" youtube: "BT, GV, FE, RB, CH-M"
allrecipes: "BT, GV, FE, RB, F68, CH-M" allrecipes: "BT, GV, FE, RB, CH-M"
amazon-search: "BT, GV, FE, RB, F68, CH-M" amazon-search: "BT, GV, FE, RB, CH-M"
bbc: "BT, GV, FE, RB, F68, CH-M" bbc: "BT, GV, FE, RB, CH-M"
bing: "BT, GV, FE, RB, F68, CH-M" bing: "BT, GV, FE, RB, CH-M"
bing-search-restaurants: "BT, GV, FE, RB, F68, CH-M" bing-search-restaurants: "BT, GV, FE, RB, CH-M"
booking: "BT, GV, FE, RB, F68, CH-M" booking: "BT, GV, FE, RB, CH-M"
cnn: "BT, GV, FE, RB, F68, CH-M" cnn: "BT, GV, FE, RB, CH-M"
cnn-ampstories: "BT, GV, FE, RB, F68, CH-M" cnn-ampstories: "BT, GV, FE, RB, CH-M"
ebay-kleinanzeigen: "BT, GV, FE, RB, F68, CH-M" ebay-kleinanzeigen: "BT, GV, FE, RB, CH-M"
ebay-kleinanzeigen-search: "BT, GV, FE, RB, F68, CH-M" ebay-kleinanzeigen-search: "BT, GV, FE, RB, CH-M"
espn: "BT, GV, FE, RB, F68, CH-M" espn: "BT, GV, FE, RB, CH-M"
facebook: "BT, GV, FE, RB, F68, CH-M" facebook: "BT, GV, FE, RB, CH-M"
facebook-cristiano: "BT, GV, FE, RB, F68, CH-M" facebook-cristiano: "BT, GV, FE, RB, CH-M"
google: "BT, GV, FE, RB, F68, CH-M" google: "BT, GV, FE, RB, CH-M"
google-maps: "BT, GV, FE, RB, F68, CH-M" google-maps: "BT, GV, FE, RB, CH-M"
google-search-restaurants: "BT, GV, FE, RB, F68, CH-M" google-search-restaurants: "BT, GV, FE, RB, CH-M"
imdb: "BT, GV, FE, RB, F68, CH-M" imdb: "BT, GV, FE, RB, CH-M"
instagram: "BT, GV, FE, RB, F68, CH-M" instagram: "BT, GV, FE, RB, CH-M"
jianshu: "BT, GV, FE, RB, F68, CH-M" jianshu: "BT, GV, FE, RB, CH-M"
microsoft-support: "BT, GV, FE, RB, F68, CH-M" microsoft-support: "BT, GV, FE, RB, CH-M"
reddit: "BT, GV, FE, RB, F68, CH-M" reddit: "BT, GV, FE, RB, CH-M"
stackoverflow: "BT, GV, FE, RB, F68, CH-M" stackoverflow: "BT, GV, FE, RB, CH-M"
web-de: "BT, GV, FE, RB, F68, CH-M" web-de: "BT, GV, FE, RB, CH-M"
wikipedia: "BT, GV, FE, RB, F68, CH-M" wikipedia: "BT, GV, FE, RB, CH-M"
youtube-watch: "BT, GV, FE, RB, F68, CH-M" youtube-watch: "BT, GV, FE, RB, CH-M"
live: live:
description: "A set of test pages that are run as live sites instead of recorded versions. These tests are available on all browsers, on all platforms." description: "A set of test pages that are run as live sites instead of recorded versions. These tests are available on all browsers, on all platforms."
benchmarks: benchmarks:

6
testing/raptor/raptor/perftest.py
View File

@ -147,11 +147,9 @@ class Perftest(object):
self.firefox_android_apps = FIREFOX_ANDROID_APPS self.firefox_android_apps = FIREFOX_ANDROID_APPS
# We are deactivating the conditioned profiles for: # We are deactivating the conditioned profiles for:
# - win10-aarch64 : no support for geckodriver see 1582757 # - win10-aarch64 : no support for geckodriver see 1582757
# - fennec_aurora: no conditioned profiles created see 1606199
# - reference browser: no conditioned profiles created see 1606767 # - reference browser: no conditioned profiles created see 1606767
self.using_condprof = not ( self.using_condprof = not (
(self.config["platform"] == "win" and self.config["processor"] == "aarch64") (self.config["platform"] == "win" and self.config["processor"] == "aarch64")
or self.config["binary"] == "org.mozilla.fennec_aurora"
or self.config["binary"] == "org.mozilla.reference.browser.raptor" or self.config["binary"] == "org.mozilla.reference.browser.raptor"
or self.config["no_conditioned_profile"] or self.config["no_conditioned_profile"]
) )
@ -161,10 +159,6 @@ class Perftest(object):
LOG.info("Using an empty profile.") LOG.info("Using an empty profile.")
self.config["using_condprof"] = self.using_condprof self.config["using_condprof"] = self.using_condprof
# We can never use e10s on fennec
if self.config["app"] == "fennec":
self.config["e10s"] = False
# To differentiate between chrome/firefox failures, we # To differentiate between chrome/firefox failures, we
# set an app variable in the logger which prefixes messages # set an app variable in the logger which prefixes messages
# with the app name # with the app name

3
testing/raptor/raptor/results.py
View File

@ -575,9 +575,6 @@ class BrowsertimeResultsHandler(PerftestResultsHandler):
and bt in ("fnbpaint", "dcf") and bt in ("fnbpaint", "dcf")
): ):
continue continue
# fennec doesn't support 'fcp'
if self.app and "fennec" in self.app.lower() and bt == "fcp":
continue
# FCP uses a different path to get the timing, so we need to do # FCP uses a different path to get the timing, so we need to do
# some checks here # some checks here

3
testing/raptor/raptor/tests/benchmarks/raptor-youtube-playback-av1-sfr.ini
View File

@ -37,8 +37,5 @@ apps = geckoview
[raptor-youtube-playback-av1-sfr-fenix] [raptor-youtube-playback-av1-sfr-fenix]
apps = fenix apps = fenix
[raptor-youtube-playback-av1-sfr-fennec68]
apps = fennec
[raptor-youtube-playback-av1-sfr-chrome] [raptor-youtube-playback-av1-sfr-chrome]
apps = chrome apps = chrome

3
testing/raptor/raptor/tests/benchmarks/raptor-youtube-playback-h264-sfr.ini
View File

@ -37,8 +37,5 @@ apps = geckoview
[raptor-youtube-playback-h264-sfr-fenix] [raptor-youtube-playback-h264-sfr-fenix]
apps = fenix apps = fenix
[raptor-youtube-playback-h264-sfr-fennec68]
apps = fennec
[raptor-youtube-playback-h264-sfr-chrome] [raptor-youtube-playback-h264-sfr-chrome]
apps = chrome apps = chrome

3
testing/raptor/raptor/tests/benchmarks/raptor-youtube-playback-hfr.ini
View File

@ -34,8 +34,5 @@ apps = geckoview
[raptor-youtube-playback-hfr-fenix] [raptor-youtube-playback-hfr-fenix]
apps = fenix apps = fenix
[raptor-youtube-playback-hfr-fennec68]
apps = fennec
[raptor-youtube-playback-hfr-chrome] [raptor-youtube-playback-hfr-chrome]
apps = chrome apps = chrome

3
testing/raptor/raptor/tests/benchmarks/raptor-youtube-playback-vp9-sfr.ini
View File

@ -37,8 +37,5 @@ apps = geckoview
[raptor-youtube-playback-vp9-sfr-fenix] [raptor-youtube-playback-vp9-sfr-fenix]
apps = fenix apps = fenix
[raptor-youtube-playback-vp9-sfr-fennec68]
apps = fennec
[raptor-youtube-playback-vp9-sfr-chrome] [raptor-youtube-playback-vp9-sfr-chrome]
apps = chrome apps = chrome

3
testing/raptor/raptor/tests/benchmarks/raptor-youtube-playback-widevine-h264-sfr.ini
View File

@ -40,8 +40,5 @@ apps = geckoview
[raptor-youtube-playback-widevine-h264-sfr-fenix] [raptor-youtube-playback-widevine-h264-sfr-fenix]
apps = fenix apps = fenix
[raptor-youtube-playback-widevine-h264-sfr-fennec68]
apps = fennec
[raptor-youtube-playback-widevine-h264-sfr-chrome] [raptor-youtube-playback-widevine-h264-sfr-chrome]
apps = chrome apps = chrome

3
testing/raptor/raptor/tests/benchmarks/raptor-youtube-playback-widevine-hfr.ini
View File

@ -40,8 +40,5 @@ apps = geckoview
[raptor-youtube-playback-widevine-hfr-fenix] [raptor-youtube-playback-widevine-hfr-fenix]
apps = fenix apps = fenix
[raptor-youtube-playback-widevine-hfr-fennec68]
apps = fennec
[raptor-youtube-playback-widevine-hfr-chrome] [raptor-youtube-playback-widevine-hfr-chrome]
apps = chrome apps = chrome

3
testing/raptor/raptor/tests/benchmarks/raptor-youtube-playback-widevine-vp9-sfr.ini
View File

@ -40,8 +40,5 @@ apps = geckoview
[raptor-youtube-playback-widevine-vp9-sfr-fenix] [raptor-youtube-playback-widevine-vp9-sfr-fenix]
apps = fenix apps = fenix
[raptor-youtube-playback-widevine-vp9-sfr-fennec68]
apps = fennec
[raptor-youtube-playback-widevine-vp9-sfr-chrome] [raptor-youtube-playback-widevine-vp9-sfr-chrome]
apps = chrome apps = chrome

2
testing/raptor/raptor/tests/benchmarks/speedometer-mobile.ini
View File

@ -9,7 +9,7 @@ type = benchmark
test_url = http://<host>:<port>/Speedometer/index.html?raptor test_url = http://<host>:<port>/Speedometer/index.html?raptor
page_cycles = 1 page_cycles = 1
page_timeout = 420000 page_timeout = 420000
apps = fenix, geckoview, refbrow, fennec, chrome-m apps = fenix, geckoview, refbrow, chrome-m
unit = score unit = score
subtest_unit = ms subtest_unit = ms
lower_is_better = false lower_is_better = false

16
testing/raptor/raptor/tests/benchmarks/youtube-playback.ini
View File

@ -32,7 +32,7 @@ subtest_lower_is_better = true
subtest_unit = score subtest_unit = score
[youtube-playback] [youtube-playback]
apps = firefox, geckoview, fenix, fennec,refbrow, chrome apps = firefox, geckoview, fenix,refbrow, chrome
alert_on = H264.1080p30@1X_dropped_frames, alert_on = H264.1080p30@1X_dropped_frames,
H264.1080p60@1X_dropped_frames, H264.1080p60@1X_dropped_frames,
H264.1440p30@1X_dropped_frames, H264.1440p30@1X_dropped_frames,
@ -58,7 +58,7 @@ alert_on = H264.1080p30@1X_dropped_frames,
test_url = http://yttest.prod.mozaws.net/2019/main.html?test_type=playbackperf-test&raptor=true&command=run&exclude=1,2&muted=true test_url = http://yttest.prod.mozaws.net/2019/main.html?test_type=playbackperf-test&raptor=true&command=run&exclude=1,2&muted=true
[youtube-playback-av1-sfr] [youtube-playback-av1-sfr]
apps = firefox, geckoview, fenix, fennec, refbrow, chrome apps = firefox, geckoview, fenix, refbrow, chrome
test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-av1-test&raptor=true&exclude=1,2&muted=true&command=run test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-av1-test&raptor=true&exclude=1,2&muted=true&command=run
[youtube-playback-h264-1080p30] [youtube-playback-h264-1080p30]
@ -82,11 +82,11 @@ preferences = {"full-screen-api.allow-trusted-requests-only": false,
"full-screen-api.warning.timeout": 0} "full-screen-api.warning.timeout": 0}
[youtube-playback-h264-sfr] [youtube-playback-h264-sfr]
apps = firefox, geckoview, fenix, fennec, refbrow, chrome apps = firefox, geckoview, fenix, refbrow, chrome
test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-h264-test&raptor=true&exclude=1,2&muted=true&command=run test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-h264-test&raptor=true&exclude=1,2&muted=true&command=run
[youtube-playback-hfr] [youtube-playback-hfr]
apps = firefox, geckoview, fenix, fennec, refbrow, chrome apps = firefox, geckoview, fenix, refbrow, chrome
alert_on = H2641080p60fps@1X_dropped_frames, alert_on = H2641080p60fps@1X_dropped_frames,
H264720p60fps@1X_dropped_frames H264720p60fps@1X_dropped_frames
test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-hfr-test&raptor=true&exclude=1,2&muted=true&command=run test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-hfr-test&raptor=true&exclude=1,2&muted=true&command=run
@ -112,25 +112,25 @@ preferences = {"full-screen-api.allow-trusted-requests-only": false,
"full-screen-api.warning.timeout": 0} "full-screen-api.warning.timeout": 0}
[youtube-playback-vp9-sfr] [youtube-playback-vp9-sfr]
apps = firefox, geckoview, fenix, fennec, refbrow, chrome apps = firefox, geckoview, fenix, refbrow, chrome
test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-vp9-test&raptor=true&exclude=1,2&muted=true&command=run test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-sfr-vp9-test&raptor=true&exclude=1,2&muted=true&command=run
[youtube-playback-widevine-h264-sfr] [youtube-playback-widevine-h264-sfr]
apps = firefox, geckoview, fenix, fennec, refbrow, chrome apps = firefox, geckoview, fenix, refbrow, chrome
test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-widevine-sfr-h264-test&raptor=true&exclude=1,2&muted=true&command=run test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-widevine-sfr-h264-test&raptor=true&exclude=1,2&muted=true&command=run
preferences = {"media.eme.enabled": true, preferences = {"media.eme.enabled": true,
"media.gmp-manager.updateEnabled": true, "media.gmp-manager.updateEnabled": true,
"media.eme.require-app-approval": false} "media.eme.require-app-approval": false}
[youtube-playback-widevine-hfr] [youtube-playback-widevine-hfr]
apps = firefox, geckoview, fenix, fennec, refbrow, chrome apps = firefox, geckoview, fenix, refbrow, chrome
test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-widevine-hfr-test&raptor=true&exclude=1,2&muted=true&command=run test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-widevine-hfr-test&raptor=true&exclude=1,2&muted=true&command=run
preferences = {"media.eme.enabled": true, preferences = {"media.eme.enabled": true,
"media.gmp-manager.updateEnabled": true, "media.gmp-manager.updateEnabled": true,
"media.eme.require-app-approval": false} "media.eme.require-app-approval": false}
[youtube-playback-widevine-vp9-sfr] [youtube-playback-widevine-vp9-sfr]
apps = firefox, geckoview, fenix, fennec, refbrow, chrome apps = firefox, geckoview, fenix, refbrow, chrome
test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-widevine-sfr-vp9-test&raptor=true&exclude=1,2&muted=true&command=run test_url = https://yttest.prod.mozaws.net/2020/main.html?test_type=playbackperf-widevine-sfr-vp9-test&raptor=true&exclude=1,2&muted=true&command=run
preferences = {"media.eme.enabled": true, preferences = {"media.eme.enabled": true,
"media.gmp-manager.updateEnabled": true, "media.gmp-manager.updateEnabled": true,

2
testing/raptor/raptor/tests/scenario/idle.ini
View File

@ -1,6 +1,6 @@
[DEFAULT] [DEFAULT]
type = scenario type = scenario
apps = fenix, geckoview, refbrow, fennec apps = fenix, geckoview, refbrow
test_url = about:blank test_url = about:blank
scenario_time = 1200000 scenario_time = 1200000
measure = fakeMeasure measure = fakeMeasure

3
testing/raptor/raptor/tests/scenario/raptor-scn-power-idle-bg.ini
View File

@ -18,6 +18,3 @@ apps = geckoview
[raptor-scn-power-idle-bg-refbrow] [raptor-scn-power-idle-bg-refbrow]
apps = refbrow apps = refbrow
[raptor-scn-power-idle-bg-fennec68]
apps = fennec

3
testing/raptor/raptor/tests/scenario/raptor-scn-power-idle.ini
View File

@ -17,6 +17,3 @@ apps = geckoview
[raptor-scn-power-idle-refbrow] [raptor-scn-power-idle-refbrow]
apps = refbrow apps = refbrow
[raptor-scn-power-idle-fennec68]
apps = fennec

2
testing/raptor/raptor/tests/tp6/live/browsertime-live.ini
View File

@ -9,7 +9,7 @@
[DEFAULT] [DEFAULT]
alert_on = fcp, loadtime alert_on = fcp, loadtime
alert_threshold = 2.0 alert_threshold = 2.0
apps = geckoview, fenix, refbrow, fennec, chrome-m, firefox, chrome, chromium apps = geckoview, fenix, refbrow, chrome-m, firefox, chrome, chromium
browser_cycles = 15 browser_cycles = 15
lower_is_better = true lower_is_better = true
page_cycles = 25 page_cycles = 25

2
testing/raptor/raptor/tests/tp6/mobile/browsertime-tp6m.ini
View File

@ -7,7 +7,7 @@
[DEFAULT] [DEFAULT]
alert_on = fcp, loadtime alert_on = fcp, loadtime
alert_threshold = 2.0 alert_threshold = 2.0
apps = geckoview, fenix, refbrow, fennec, chrome-m apps = geckoview, fenix, refbrow, chrome-m
browser_cycles = 15 browser_cycles = 15
lower_is_better = true lower_is_better = true
page_cycles = 25 page_cycles = 25

51
testing/raptor/raptor/webextension/android.py
View File

@ -75,9 +75,6 @@ class WebExtensionAndroid(PerftestAndroid, WebExtension):
# on-device: /data/local/tmp/com.yourcompany.yourapp-geckoview-config.yaml # on-device: /data/local/tmp/com.yourcompany.yourapp-geckoview-config.yaml
# https://mozilla.github.io/geckoview/tutorials/automation.html#configuration-file-format # https://mozilla.github.io/geckoview/tutorials/automation.html#configuration-file-format
# only supported for geckoview apps
if self.config["app"] == "fennec":
return
LOG.info("creating android app config.yml") LOG.info("creating android app config.yml")
yml_config_data = dict( yml_config_data = dict(
@ -134,49 +131,21 @@ class WebExtensionAndroid(PerftestAndroid, WebExtension):
def launch_firefox_android_app(self, test_name): def launch_firefox_android_app(self, test_name):
LOG.info("starting %s" % self.config["app"]) LOG.info("starting %s" % self.config["app"])
extra_args = [
"-profile",
self.remote_profile,
"--allow-downgrade",
"--es",
"env0",
"LOG_VERBOSE=1",
"--es",
"env1",
"R_LOG_LEVEL=6",
"--es",
"env2",
"MOZ_WEBRENDER=%d" % self.config["enable_webrender"],
# Force the app to immediately exit for content crashes
"--es",
"env3",
"MOZ_CRASHREPORTER_SHUTDOWN=1",
]
try: try:
# make sure the android app is not already running # make sure the android app is not already running
self.device.stop_application(self.config["binary"]) self.device.stop_application(self.config["binary"])
if self.config["app"] == "fennec": # command line 'extra' args not used with geckoview apps; instead we use
self.device.launch_fennec( # an on-device config.yml file (see write_android_app_config)
self.config["binary"],
extra_args=extra_args,
url="about:blank",
fail_if_running=False,
)
else:
# command line 'extra' args not used with geckoview apps; instead we use self.device.launch_application(
# an on-device config.yml file (see write_android_app_config) self.config["binary"],
self.config["activity"],
self.device.launch_application( self.config["intent"],
self.config["binary"], extras=None,
self.config["activity"], url="about:blank",
self.config["intent"], fail_if_running=False,
extras=None, )
url="about:blank",
fail_if_running=False,
)
# Check if app has started and it's running # Check if app has started and it's running
if not self.process_exists: if not self.process_exists:

2
testing/raptor/raptor/webextension/base.py
View File

@ -196,7 +196,7 @@ class WebExtension(Perftest):
return return
LOG.info("removing webext %s" % self.raptor_webext) LOG.info("removing webext %s" % self.raptor_webext)
if self.config["app"] in ["firefox", "geckoview", "fennec", "refbrow", "fenix"]: if self.config["app"] in ["firefox", "geckoview", "refbrow", "fenix"]:
self.profile.addons.remove_addon(self.webext_id) self.profile.addons.remove_addon(self.webext_id)
# for chrome the addon is just a list (appended to cmd line) # for chrome the addon is just a list (appended to cmd line)

5
testing/raptor/test/test_raptor.py
View File

@ -59,12 +59,10 @@ class TestBrowserThread(threading.Thread):
[WebExtensionFirefox, "firefox"], [WebExtensionFirefox, "firefox"],
[WebExtensionDesktopChrome, "chrome"], [WebExtensionDesktopChrome, "chrome"],
[WebExtensionDesktopChrome, "chromium"], [WebExtensionDesktopChrome, "chromium"],
[WebExtensionAndroid, "fennec"],
[WebExtensionAndroid, "geckoview"], [WebExtensionAndroid, "geckoview"],
[BrowsertimeDesktop, "firefox"], [BrowsertimeDesktop, "firefox"],
[BrowsertimeDesktop, "chrome"], [BrowsertimeDesktop, "chrome"],
[BrowsertimeDesktop, "chromium"], [BrowsertimeDesktop, "chromium"],
[BrowsertimeAndroid, "fennec"],
[BrowsertimeAndroid, "geckoview"], [BrowsertimeAndroid, "geckoview"],
], ],
) )
@ -104,7 +102,7 @@ def test_perftest_host_ip(ConcretePerftest, options, get_prefs):
@pytest.mark.parametrize( @pytest.mark.parametrize(
"app_name, expected_e10s_flag", "app_name, expected_e10s_flag",
[["firefox", True], ["fennec", False], ["geckoview", True]], [["firefox", True], ["geckoview", True]],
) )
def test_e10s_enabling(ConcretePerftest, options, app_name, expected_e10s_flag): def test_e10s_enabling(ConcretePerftest, options, app_name, expected_e10s_flag):
options["app"] = app_name options["app"] = app_name
@ -123,7 +121,6 @@ def test_profile_was_provided_locally(ConcretePerftest, options):
["firefox", "firefox", "firefox"], ["firefox", "firefox", "firefox"],
[None, "firefox", "firefox"], [None, "firefox", "firefox"],
["firefox", None, "firefox"], ["firefox", None, "firefox"],
["firefox", "fennec", "firefox"],
], ],
) )
def test_profile_class_assignation( def test_profile_class_assignation(

2
testing/raptor/webext/raptor/runner.js
View File

@ -19,7 +19,7 @@ const TEST_BENCHMARK = "benchmark";
const TEST_PAGE_LOAD = "pageload"; const TEST_PAGE_LOAD = "pageload";
const TEST_SCENARIO = "scenario"; const TEST_SCENARIO = "scenario";
const ANDROID_BROWSERS = ["fennec", "fenix", "geckoview", "refbrow"]; const ANDROID_BROWSERS = ["fenix", "geckoview", "refbrow"];
// when the browser starts this webext runner will start automatically; we // when the browser starts this webext runner will start automatically; we
// want to give the browser some time (ms) to settle before starting tests // want to give the browser some time (ms) to settle before starting tests