Bug 1607891 - Add fennec (68) support to raptor-browsertime r=tarek

Differential Revision: https://phabricator.services.mozilla.com/D59737

--HG--
extra : moz-landing-system : lando

taskcluster/ci/test/browsertime-mobile.yml

@@ -16,6 +16,10 @@ job-defaults:
                 by-test-platform:
                     android-hw-p2-.*aarch64.*/pgo: ["mozilla-central", "try"]
                     default: []
+            fennec:
+                by-test-platform:
+                    android-hw-p2-.*aarch64.*/pgo: ["mozilla-central", "try"]
+                    default: []
             refbrow: []
     tier: 3
     target:
@@ -28,6 +32,14 @@ job-defaults:
                     default:
                         index: project.mobile.fenix.v2.performance-test.latest
                         name: build/armeabi-v7a/geckoNightly/target.apk
+            fennec:
+                by-test-platform:
+                    android-hw-p2-8-0-android-aarch64.*:
+                        index: gecko.v2.mozilla-esr68.latest.mobile.android-aarch64-opt
+                        name: build/target.apk
+                    default:
+                        index: gecko.v2.mozilla-esr68.latest.mobile.android-api-16-opt
+                        name: build/target.apk
             geckoview: geckoview_example.apk
             refbrow:
                 by-test-platform:
@@ -43,11 +55,13 @@ job-defaults:
         by-app:
             fenix: org.mozilla.fenix.IntentReceiverActivity
             geckoview: org.mozilla.geckoview_example.GeckoViewActivity
+            fennec: org.mozilla.gecko.BrowserApp
             refbrow: org.mozilla.reference.browser.BrowserTestActivity
     binary-path:
         by-app:
             fenix: org.mozilla.fenix.performancetest
             geckoview: org.mozilla.geckoview_example
+            fennec: org.mozilla.fennec_aurora
             refbrow: org.mozilla.reference.browser.raptor
     mozharness:
         script: raptor_script.py
@@ -60,7 +74,7 @@ browsertime-tp6m:
     description: "Raptor (browsertime) tp6 page-load tests on android"
     raptor-test: tp6m
     raptor-subtests: ['amazon', 'youtube']
-    apps: ["geckoview", "fenix"]
+    apps: ["geckoview", "fenix", "fennec"]
     pageload: cold
     run-visual-metrics: true
     treeherder-symbol: Btime(tp6m)

testing/mozharness/mozharness/mozilla/testing/raptor.py

@@ -491,6 +491,8 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin):
         # Options overwritten from **kw
         if 'test' in self.config:
             kw_options['test'] = self.config['test']
+        if 'binary' in self.config:
+            kw_options['binary'] = self.config['binary']
         if self.symbols_path:
             kw_options['symbolsPath'] = self.symbols_path
         if self.config.get('obj_path', None) is not None:
@@ -499,6 +501,8 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin):
             kw_options['test-url-params'] = self.test_url_params
         if self.config.get('device_name') is not None:
             kw_options['device-name'] = self.config['device_name']
+        if self.config.get('activity') is not None:
+            kw_options['activity'] = self.config['activity']
 
         kw_options.update(kw)
         if self.host:

testing/raptor/raptor/manifest.py

@@ -453,6 +453,13 @@ def get_raptor_test_list(args, oskey):
                 next_test['page_timeout']) * LIVE_SITE_TIMEOUT_MULTIPLIER
             LOG.info("using live sites so using page timeout of %dms" % next_test['page_timeout'])
 
+        # browsertime doesn't use the 'measure' test ini setting; however just for the sake
+        # of supporting both webext and browsertime, just provide a dummy 'measure' setting
+        # here to prevent having to check in multiple places; it has no effect on what
+        # browsertime actually measures; remove this when eventually we remove webext support
+        if args.browsertime is not None and next_test.get('measure') is None:
+            next_test['measure'] = "fnbpaint, fcp, dcf, loadtime"
+
         # convert 'measure =' test INI line to list
         if next_test.get('measure') is not None:
             _measures = []

testing/raptor/raptor/results.py

@@ -431,6 +431,9 @@ class BrowsertimeResultsHandler(PerftestResultsHandler):
                     # chrome we just measure fcp and loadtime; skip fnbpaint and dcf
                     if self.app and 'chrome' in self.app.lower() and bt in ('fnbpaint', 'dcf'):
                         continue
+                    # fennec doesn't support 'fcp'
+                    if self.app and 'fennec' in self.app.lower() and bt == 'fcp':
+                        continue
 
                     # chrome currently uses different names (and locations) for some metrics
                     if raptor in chrome_raptor_conversion and \

testing/raptor/raptor/tests/tp6/mobile/browsertime-tp6m.ini

@@ -7,10 +7,9 @@
 [DEFAULT]
 alert_on = fcp, loadtime
 alert_threshold = 2.0
-apps = geckoview, fenix, refbrow
+apps = geckoview, fenix, refbrow, fennec
 browser_cycles = 15
 lower_is_better = true
-measure = fnbpaint, fcp, dcf, loadtime
 page_cycles = 25
 page_timeout = 60000
 playback = mitmproxy-android