Bug 1480841 - Raptor support for running speedometer on geckoview r=jmaher

Differential Revision: https://phabricator.services.mozilla.com/D3446

--HG--
extra : moz-landing-system : lando

testing/mozharness/mozharness/mozilla/testing/raptor.py

@@ -55,7 +55,7 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin):
           }],
         [["--app"],
          {"default": "firefox",
-          "choices": ["firefox", "chrome"],
+          "choices": ["firefox", "chrome", "geckoview"],
           "dest": "app",
           "help": "name of the application we are testing (default: firefox)"
           }],
@@ -109,12 +109,16 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin):
         if self.run_local:
             # raptor initiated locally, get app from command line args
             # which are passed in from mach inside 'raptor_cmd_line_args'
+            # cmd line args can be in two formats depending on how user entered them
+            # i.e. "--app=geckoview" or separate as "--app", "geckoview" so we have to
+            # check each cmd line arg individually
             self.app = "firefox"
             if 'raptor_cmd_line_args' in self.config:
-                for next_arg in self.config['raptor_cmd_line_args']:
-                    if "chrome" in next_arg:
-                        self.app = "chrome"
-                        break
+                for app in ['chrome', 'geckoview']:
+                    for next_arg in self.config['raptor_cmd_line_args']:
+                        if app in next_arg:
+                            self.app = app
+                            break
         else:
             # raptor initiated in production via mozharness
             self.test = self.config['test']

testing/profiles/raptor/user.js

@@ -1,3 +1,6 @@
 // Preferences file used by the raptor harness
 /* globals user_pref */
 user_pref("dom.performance.time_to_non_blank_paint.enabled", true);
+
+// required for geckoview logging
+user_pref("geckoview.console.enabled", true);

testing/raptor/raptor/cmdline.py

@@ -17,7 +17,7 @@ def create_parser(mach_interface=False):
             help="name of raptor test to run")
     add_arg('--app', default='firefox', dest='app',
             help="name of the application we are testing (default: firefox)",
-            choices=['firefox', 'chrome'])
+            choices=['firefox', 'chrome', 'geckoview'])
     add_arg('-b', '--binary', dest='binary',
             help="path to the browser executable that we are testing")
     if not mach_interface:
@@ -39,8 +39,10 @@ def verify_options(parser, args):
     if args.binary is None:
         parser.error("--binary is required!")
 
-    if not os.path.isfile(args.binary):
-        parser.error("{binary} does not exist!".format(**ctx))
+    # if running on a desktop browser make sure the binary exists
+    if args.app != "geckoview":
+        if not os.path.isfile(args.binary):
+            parser.error("{binary} does not exist!".format(**ctx))
 
 
 def parse_args(argv=None):

testing/raptor/raptor/control_server.py

@@ -88,6 +88,8 @@ class RaptorControlServer():
         self.results_handler = results_handler
         self.browser_proc = None
         self._finished = False
+        self.device = None
+        self.app_name = None
 
     def start(self):
         config_dir = os.path.join(here, 'tests')
@@ -112,7 +114,10 @@ class RaptorControlServer():
         self.server = httpd
 
     def shutdown_browser(self):
-        LOG.info("shutting down browser (pid: %d)" % self.browser_proc.pid)
+        if self.device is not None:
+            LOG.info("shutting down android app %s" % self.app_name)
+        else:
+            LOG.info("shutting down browser (pid: %d)" % self.browser_proc.pid)
         self.kill_thread = threading.Thread(target=self.wait_for_quit)
         self.kill_thread.daemon = True
         self.kill_thread.start()
@@ -121,9 +126,12 @@ class RaptorControlServer():
         """Wait timeout seconds for the process to exit. If it hasn't
         exited by then, kill it.
         """
-        self.browser_proc.wait(timeout)
-        if self.browser_proc.poll() is None:
-            self.browser_proc.kill()
+        if self.device is not None:
+            self.device.stop_application(self.app_name)
+        else:
+            self.browser_proc.wait(timeout)
+            if self.browser_proc.poll() is None:
+                self.browser_proc.kill()
         self._finished = True
 
     def stop(self):

testing/raptor/raptor/raptor.py

@@ -12,6 +12,7 @@ import time
 
 import mozinfo
 
+from mozdevice import ADBAndroid
 from mozlog import commandline, get_default_logger
 from mozprofile import create_profile
 from mozrunner import runners
@@ -54,8 +55,11 @@ class Raptor(object):
         self.playback = None
         self.benchmark = None
 
-        # Create the profile
-        self.profile = create_profile(self.config['app'])
+        # Create the profile; for geckoview we want a firefox profile type
+        if self.config['app'] == 'geckoview':
+            self.profile = create_profile('firefox')
+        else:
+            self.profile = create_profile(self.config['app'])
 
         # Merge in base profiles
         with open(os.path.join(self.profile_data_dir, 'profiles.json'), 'r') as fh:
@@ -69,14 +73,24 @@ class Raptor(object):
         # create results holder
         self.results_handler = RaptorResultsHandler()
 
-        # Create the runner
-        self.output_handler = OutputHandler()
-        process_args = {
-            'processOutputLine': [self.output_handler],
-        }
-        runner_cls = runners[app]
-        self.runner = runner_cls(
-            binary, profile=self.profile, process_args=process_args)
+        # when testing desktop browsers we use mozrunner to start the browser; when
+        # testing on android (i.e. geckoview) we use mozdevice to control the device app
+
+        if self.config['app'] == "geckoview":
+            # create the android device handler; it gets initiated and sets up adb etc
+            self.log.info("creating android device handler using mozdevice")
+            self.device = ADBAndroid(verbose=True)
+            self.device.clear_logcat()
+        else:
+            # create the desktop browser runner
+            self.log.info("creating browser runner using mozrunner")
+            self.output_handler = OutputHandler()
+            process_args = {
+                'processOutputLine': [self.output_handler],
+            }
+            runner_cls = runners[app]
+            self.runner = runner_cls(
+                binary, profile=self.profile, process_args=process_args)
 
         self.log.info("raptor config: %s" % str(self.config))
 
@@ -92,6 +106,13 @@ class Raptor(object):
         self.control_server = RaptorControlServer(self.results_handler)
         self.control_server.start()
 
+        # for android we must make the control server available to the device
+        if self.config['app'] == "geckoview":
+            self.log.info("making the raptor control server port available to device")
+            _tcp_port = "tcp:%s" % self.control_server.port
+            _cmd = ["reverse", _tcp_port, _tcp_port]
+            self.device.command_output(_cmd)
+
     def get_playback_config(self, test):
         self.config['playback_tool'] = test.get('playback')
         self.log.info("test uses playback tool: %s " % self.config['playback_tool'])
@@ -121,6 +142,13 @@ class Raptor(object):
                         self.control_server.port,
                         benchmark_port)
 
+        # for android we must make the benchmarks server available to the device
+        if self.config['app'] == "geckoview":
+            self.log.info("making the raptor benchmarks server port available to device")
+            _tcp_port = "tcp:%s" % benchmark_port
+            _cmd = ["reverse", _tcp_port, _tcp_port]
+            self.device.command_output(_cmd)
+
         # must intall raptor addon each time because we dynamically update some content
         raptor_webext = os.path.join(webext_dir, 'raptor')
         self.log.info("installing webext %s" % raptor_webext)
@@ -135,7 +163,7 @@ class Raptor(object):
                               but we do not install them on non Firefox browsers.")
 
         # on firefox we can get an addon id; chrome addon actually is just cmd line arg
-        if self.config['app'] == "firefox":
+        if self.config['app'] in ["firefox", "geckoview"]:
             webext_id = self.profile.addons.addon_details(raptor_webext)['id']
 
         # some tests require tools to playback the test pages
@@ -144,11 +172,46 @@ class Raptor(object):
             # startup the playback tool
             self.playback = get_playback(self.config)
 
-        self.runner.start()
+        # for geckoview we must copy the profile onto the device and set perms
+        if self.config['app'] == "geckoview":
+            self.log.info("copying firefox profile onto the android device")
+            self.device_profile = "/sdcard/raptor-profile"
+            if self.device.is_dir(self.device_profile):
+                self.device.rm(self.device_profile, recursive=True)
 
-        proc = self.runner.process_handler
-        self.output_handler.proc = proc
-        self.control_server.browser_proc = proc
+            self.device.mkdir(self.device_profile)
+            self.device.push(self.profile.profile, self.device_profile)
+
+            self.log.info("setting permisions to profile dir on the device")
+            self.device.chmod(self.device_profile, recursive=True)
+
+            # now start the geckoview app
+            self.log.info("starting %s" % self.config['app'])
+
+            extra_args = ["-profile", self.device_profile,
+                          "--es", "env0", "LOG_VERBOSE=1",
+                          "--es", "env1", "R_LOG_LEVEL=6"]
+
+            self.device.launch_activity(self.config['binary'],
+                                        "GeckoViewActivity",
+                                        extra_args=extra_args,
+                                        url='about:blank',
+                                        fail_if_running=False)
+
+            self.control_server.device = self.device
+            self.control_server.app_name = self.config['binary']
+
+        else:
+            # now start the desktop browser
+            self.log.info("starting %s" % self.config['app'])
+
+            self.runner.start()
+            proc = self.runner.process_handler
+            self.output_handler.proc = proc
+
+            self.control_server.browser_proc = proc
+
+        # set our cs flag to indicate we are running the browser/app
         self.control_server._finished = False
 
         # convert to seconds and account for page cycles
@@ -159,26 +222,31 @@ class Raptor(object):
                 time.sleep(1)
                 elapsed_time += 1
                 if elapsed_time > (timeout) - 5:  # stop 5 seconds early
+                    self.log.info("application timed out after {} seconds".format(timeout))
                     self.control_server.wait_for_quit()
                     break
         finally:
-            try:
-                self.runner.check_for_crashes()
-            except NotImplementedError:  # not implemented for Chrome
-                pass
+            if self.config['app'] != "geckoview":
+                try:
+                    self.runner.check_for_crashes()
+                except NotImplementedError:  # not implemented for Chrome
+                    pass
+            # TODO: if on geckoview is there some cleanup here i.e. check for crashes?
 
         if self.playback is not None:
             self.playback.stop()
 
         # remove the raptor webext; as it must be reloaded with each subtest anyway
         # applies to firefox only; chrome the addon is actually just cmd line arg
-        if self.config['app'] == "firefox":
+        if self.config['app'] in ["firefox", "geckoview"]:
             self.log.info("removing webext %s" % raptor_webext)
             self.profile.addons.remove_addon(webext_id)
 
-        if self.runner.is_running():
-            self.log.info("Application timed out after {} seconds".format(timeout))
-            self.runner.stop()
+        if self.config['app'] != "geckoview":
+            if self.runner.is_running():
+                self.runner.stop()
+        # TODO the geckoview app should have been shutdown by this point by the
+        # control server, but we can double-check here to make sure
 
     def process_results(self):
         # when running locally output results in build/raptor.json; when running
@@ -197,7 +265,8 @@ class Raptor(object):
 
     def clean_up(self):
         self.control_server.stop()
-        self.runner.stop()
+        if self.config['app'] != "geckoview":
+            self.runner.stop()
         self.log.info("finished")
 
 

testing/raptor/raptor/tests/raptor-speedometer.ini

@@ -18,3 +18,8 @@ apps = firefox
 
 [raptor-speedometer-chrome]
 apps = chrome
+
+[raptor-speedometer-geckoview]
+page_timeout = 1800000 # temp 30 min for debugging
+page_cycles = 1
+apps = geckoview

testing/raptor/requirements.txt

@@ -2,3 +2,4 @@ mozrunner ~= 7.0
 mozprofile ~= 1.1
 manifestparser >= 1.1
 wptserve ~= 1.4.0
+mozdevice ~= 1.0

testing/raptor/test/test_cmdline.py

@@ -10,7 +10,7 @@ from raptor.cmdline import verify_options
 
 
 def test_verify_options(filedir):
-    args = Namespace(binary='invalid/path')
+    args = Namespace(app='firefox', binary='invalid/path')
     parser = ArgumentParser()
 
     with pytest.raises(SystemExit):

testing/raptor/webext/raptor/benchmark-relay.js

@@ -22,4 +22,5 @@ function sendResult(_type, _value) {
   });
 }
 
+console.log("raptor benchmark-relay content loaded");
 window.addEventListener("message", receiveMessage);

testing/raptor/webext/raptor/runner.js

@@ -1,3 +1,4 @@
+
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
@@ -105,7 +106,7 @@ function getTestSettings() {
         }
 
         // write options to storage that our content script needs to know
-        if (browserName === "firefox") {
+        if (["firefox", "geckoview"].includes(browserName)) {
           ext.storage.local.clear().then(function() {
             ext.storage.local.set({settings}).then(function() {
               console.log("wrote settings to ext local storage");
@@ -127,7 +128,7 @@ function getTestSettings() {
 
 function getBrowserInfo() {
   return new Promise(resolve => {
-    if (browserName === "firefox") {
+    if (["firefox", "geckoview"].includes(browserName)) {
       ext = browser;
       var gettingInfo = browser.runtime.getBrowserInfo();
       gettingInfo.then(function(bi) {
@@ -157,7 +158,7 @@ function testTabCreated(tab) {
 }
 
 async function testTabUpdated(tab) {
-  console.log("tab " + tab.id + " reloaded");
+  console.log("test tab updated");
   // wait for pageload test result from content
   await waitForResult();
   // move on to next cycle (or test complete)
@@ -215,7 +216,7 @@ function nextCycle() {
       } else if (testType == "benchmark") {
         isBenchmarkPending = true;
       }
-      // (re)load the test page
+      // update the test page - browse to our test URL
       ext.tabs.update(testTabID, {url: testURL}, testTabUpdated);
     }, pageCycleDelay);
   } else {
@@ -241,7 +242,7 @@ function setTimeoutAlarm(timeoutName, timeoutMS) {
 }
 
 function cancelTimeoutAlarm(timeoutName) {
-  if (browserName === "firefox") {
+  if (browserName === "firefox" || browserName === "geckoview") {
     var clearAlarm = ext.alarms.clear(timeoutName);
     clearAlarm.then(function(onCleared) {
       if (onCleared) {
@@ -359,6 +360,7 @@ function cleanUp() {
 }
 
 function runner() {
+  console.log("Welcome to Jurassic Park!");
   let config = getTestConfig();
   console.log("test name is: " + config.test_name);
   console.log("test settings url is: " + config.test_settings_url);
@@ -388,7 +390,13 @@ function runner() {
       // wait some time for the browser to settle before beginning
       var text = "* pausing " + postStartupDelay / 1000 + " seconds to let browser settle... *";
       postToControlServer("status", text);
-      setTimeout(function() { ext.tabs.create({url: "about:blank"}); }, postStartupDelay);
+
+      // on geckoview you can't create a new tab; only using existing tab - set it blank first
+      if (config.browser == "geckoview") {
+        setTimeout(function() { nextCycle(); }, postStartupDelay);
+      } else {
+        setTimeout(function() { ext.tabs.create({url: "about:blank"}); }, postStartupDelay);
+      }
     });
   });
 }

third_party/webkit/PerformanceTests/Speedometer/resources/benchmark-report.js

@@ -88,6 +88,7 @@
 
                 if (location.search == '?raptor') {
                     _data = ['raptor-benchmark', 'speedometer', measuredValuesByFullName];
+                    console.log('speedometer source about to post results to the raptor webext');
                     window.postMessage(_data, '*');
                 } else {
                     tpRecordTime(values.join(','), 0, allNames.join(','));