Bug 1601009 - Add support to raptor-browsertime for running the speedometer benchmark on Firefox android r=tarek

Differential Revision: https://phabricator.services.mozilla.com/D61305

--HG--
extra : moz-landing-system : lando

taskcluster/ci/test/browsertime-mobile.yml

@@ -96,3 +96,12 @@ browsertime-tp6m-profiling:
         extra-options:
             - --browsertime
             - --gecko-profile
+
+browsertime-benchmark-speedometer:
+    description: "Browsertime speedometer on android"
+    apps: ["geckoview", "fenix", "fennec"]
+    treeherder-symbol: Btime(sp)
+    mozharness:
+        extra-options:
+            - --browsertime
+            - --test=raptor-speedometer

taskcluster/ci/test/test-sets.yml

@@ -994,7 +994,9 @@ android-hw-aarch64-raptor-cpu-memory-power:
 android-hw-aarch64-browsertime:
     - browsertime-tp6m
     - browsertime-tp6m-profiling
+    - browsertime-benchmark-speedometer
 
 android-hw-arm7-browsertime:
     - browsertime-tp6m
     - browsertime-tp6m-profiling
+    - browsertime-benchmark-speedometer

testing/raptor/raptor/browsertime/android.py

@@ -100,7 +100,9 @@ class BrowsertimeAndroid(PerftestAndroid, Browsertime):
         super(BrowsertimeAndroid, self).run_test_setup(test)
 
         self.set_reverse_ports()
-        self.turn_on_android_app_proxy()
+
+        if self.playback:
+            self.turn_on_android_app_proxy()
         self.remove_mozprofile_delimiters_from_profile()
 
     def run_tests(self, tests, test_names):

testing/raptor/raptor/browsertime/base.py

@@ -19,7 +19,10 @@ from perftest import Perftest
 from results import BrowsertimeResultsHandler
 
 LOG = RaptorLogger(component="raptor-browsertime")
+
 DEFAULT_CHROMEVERSION = "77"
+BROWSERTIME_PAGELOAD_OUTPUT_TIMEOUT = 120  # 2 minutes
+BROWSERTIME_BENCHMARK_OUTPUT_TIMEOUT = 900  # 15 minutes
 
 
 class Browsertime(Perftest):
@@ -291,7 +294,9 @@ class Browsertime(Perftest):
             proc = self.process_handler(cmd, env=env)
             proc.run(
                 timeout=self._compute_process_timeout(test, timeout),
-                outputTimeout=2 * 60,
+                outputTimeout=BROWSERTIME_BENCHMARK_OUTPUT_TIMEOUT
+                if self.benchmark
+                else BROWSERTIME_PAGELOAD_OUTPUT_TIMEOUT,
             )
             proc.wait()
 

testing/raptor/raptor/perftest.py

@@ -144,7 +144,6 @@ either Raptor or browsertime."""
         self.installerpath = installerpath
         self.playback = None
         self.benchmark = None
-        self.benchmark_port = 0
         self.gecko_profiler = None
         self.post_startup_delay = post_startup_delay
         self.device = None
@@ -512,9 +511,9 @@ class PerftestAndroid(Perftest):
 
             if self.benchmark:
                 LOG.info("making the raptor benchmarks server port available to device")
-                self.set_reverse_port(self.benchmark_port)
+                self.set_reverse_port(int(self.benchmark.port))
         else:
-            LOG.info("Reverse port forwarding is uded only on local devices")
+            LOG.info("Reverse port forwarding is used only on local devices")
 
     def build_browser_profile(self):
         super(PerftestAndroid, self).build_browser_profile()

testing/raptor/raptor/webextension/base.py

@@ -66,7 +66,7 @@ class WebExtension(Perftest):
             self.control_server.port,
             self.post_startup_delay,
             host=self.config["host"],
-            b_port=self.benchmark_port,
+            b_port=int(self.benchmark.port) if self.benchmark else 0,
             debug_mode=1 if self.debug_mode else 0,
             browser_cycle=test.get("browser_cycle", 1),
         )
@@ -154,7 +154,6 @@ class WebExtension(Perftest):
     def serve_benchmark_source(self, test):
         # benchmark-type tests require the benchmark test to be served out
         self.benchmark = Benchmark(self.config, test)
-        self.benchmark_port = int(self.benchmark.port)
 
     def install_raptor_webext(self):
         # must intall raptor addon each time because we dynamically update some content

testing/raptor/test/test_raptor.py

@@ -326,7 +326,7 @@ def test_android_reverse_ports(host, playback, benchmark):
     if benchmark:
         benchmark_mock = mock.patch("raptor.raptor.benchmark.Benchmark")
         raptor.benchmark = benchmark_mock
-        raptor.benchmark_port = 1234
+        raptor.benchmark.port = 1234
 
     if playback:
         playback_mock = mock.patch(