Bug 1639372 - Run visual metrics locally through raptor. r=tarek,perftest-reviewers,Bebe

This patch makes it possible to run visual metrics through raptor-browsertime locally.

Differential Revision: https://phabricator.services.mozilla.com/D90548

testing/mozharness/mozharness/mozilla/testing/raptor.py

@@ -70,6 +70,11 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin, Pyt
             "default": None,
             "help": argparse.SUPPRESS
         }],
+        [["--browsertime-vismet-script"], {
+            "dest": "browsertime_vismet_script",
+            "default": None,
+            "help": argparse.SUPPRESS
+        }],
         [["--browsertime-chromedriver"], {
             "dest": "browsertime_chromedriver",
             "default": None,
@@ -91,6 +96,12 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin, Pyt
             "default": False,
             "help": argparse.SUPPRESS
         }],
+        [["--browsertime-visualmetrics"], {
+            "dest": "browsertime_visualmetrics",
+            "action": "store_true",
+            "default": False,
+            "help": argparse.SUPPRESS
+        }],
         [["--browsertime-no-ffwindowrecorder"], {
             "dest": "browsertime_no_ffwindowrecorder",
             "action": "store_true",
@@ -418,6 +429,8 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin, Pyt
         self.chromium_dist_path = None
         self.firefox_android_browsers = ["fennec", "geckoview", "refbrow", "fenix"]
         self.android_browsers = self.firefox_android_browsers + ["chrome-m"]
+        self.browsertime_visualmetrics = False
+        self.browsertime_video = False
 
         for (arg,), details in Raptor.browsertime_options:
             # Allow overriding defaults on the `./mach raptor-test ...` command-line.
@@ -425,6 +438,9 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin, Pyt
             if value and arg not in self.config.get("raptor_cmd_line_args", []):
                 setattr(self, details['dest'], value)
 
+        if not self.run_local and self.browsertime_visualmetrics and self.browsertime_video:
+            self.error("Cannot run visual metrics in the same CI task as the test.")
+
     # We accept some configuration options from the try commit message in the
     # format mozharness: <options>. Example try commit message: mozharness:
     # --geckoProfile try: <stuff>
@@ -746,10 +762,20 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin, Pyt
             two_pass=True,
             editable=True,
         )
+
+        modules = ['pip>=1.5']
+        if self.run_local:
+            # Add modules required for visual metrics
+            modules.extend([
+                'numpy==1.16.1',
+                'Pillow==6.1.0',
+                'scipy==1.2.3',
+                'pyssim==0.4'
+            ])
+
         # Require pip >= 1.5 so pip will prefer .whl files to install
-        super(Raptor, self).create_virtualenv(
-            modules=['pip>=1.5']
-        )
+        super(Raptor, self).create_virtualenv(modules=modules)
+
         # Install Raptor dependencies
         self.install_module(
             requirements=[os.path.join(self.raptor_path,

testing/raptor/mach_commands.py

@@ -181,7 +181,30 @@ class RaptorRunner(MozbuildObject):
             self.config.update({
                 'browsertime_node': browsertime.node_path(),
                 'browsertime_browsertimejs': browsertime.browsertime_path(),
+                'browsertime_vismet_script': browsertime.visualmetrics_path(),
             })
+
+            def _browsertime_exists():
+                return (
+                    os.path.exists(self.config["browsertime_browsertimejs"]) and
+                    os.path.exists(self.config["browsertime_vismet_script"])
+                )
+            # Check if browsertime scripts exist and try to install them if
+            # they aren't
+            if not _browsertime_exists():
+                # TODO: Make this "integration" nicer in the near future
+                print("Missing browsertime files...attempting to install")
+                subprocess.check_call([
+                    os.path.join(self.topsrcdir, "mach"),
+                    "browsertime",
+                    "--setup",
+                    "--clobber"
+                ])
+                if not _browsertime_exists():
+                    raise Exception(
+                        "Failed installation attempt. Cannot find browsertime scripts. "
+                        "Run `./mach browsertime --setup --clobber` to set it up."
+                    )
         finally:
             sys.path = sys.path[1:]
 

testing/raptor/raptor/browsertime/base.py

@@ -12,6 +12,7 @@ import os
 import json
 import re
 import six
+import sys
 
 import mozprocess
 from benchmark import Benchmark
@@ -56,6 +57,9 @@ class Browsertime(Perftest):
         LOG.info("cwd: '{}'".format(os.getcwd()))
         self.config["browsertime"] = True
 
+        # Setup browsertime-specific settings for result parsing
+        self.results_handler.browsertime_visualmetrics = self.browsertime_visualmetrics
+
         # For debugging.
         for k in (
             "browsertime_node",
@@ -215,7 +219,6 @@ class Browsertime(Perftest):
             "--firefox.disableBrowsertimeExtension", "true",
             "--pageCompleteCheckStartWait", "5000",
             "--pageCompleteCheckPollTimeout", "1000",
-            "--visualMetrics", "false",
             # url load timeout (milliseconds)
             "--timeouts.pageLoad", str(timeout),
             # running browser scripts timeout (milliseconds)
@@ -231,7 +234,8 @@ class Browsertime(Perftest):
             # recorder.  In the future we'd like to be able to selectively use Android's `adb
             # screenrecord` as well.  (There's no harm setting Firefox options for other browsers.)
             browsertime_options.extend([
-                "--video", "true"
+                "--video", "true",
+                "--visualMetrics", "true" if self.browsertime_visualmetrics else "false",
             ])
 
             if self.browsertime_no_ffwindowrecorder:
@@ -247,6 +251,7 @@ class Browsertime(Perftest):
         else:
             browsertime_options.extend([
                 "--video", "false",
+                "--visualMetrics", "false"
             ])
 
         # have browsertime use our newly-created conditioned-profile path
@@ -381,6 +386,29 @@ class Browsertime(Perftest):
                 else:
                     LOG.info(msg)
 
+            if self.browsertime_visualmetrics and self.run_local:
+                # Check if visual metrics is installed correctly before running the test
+                self.vismet_failed = False
+
+                def _vismet_line_handler(line):
+                    LOG.info(line)
+                    if "FAIL" in line:
+                        self.vismet_failed = True
+
+                proc = self.process_handler(
+                    [sys.executable, self.browsertime_vismet_script, "--check"],
+                    processOutputLine=_vismet_line_handler,
+                    env=env
+                )
+                proc.run()
+                proc.wait()
+
+                if self.vismet_failed:
+                    raise Exception(
+                        "Browsertime visual metrics dependencies were not "
+                        "installed correctly."
+                    )
+
             proc = self.process_handler(cmd, processOutputLine=_line_handler, env=env)
             proc.run(
                 timeout=self._compute_process_timeout(test, timeout),

testing/raptor/raptor/cmdline.py

@@ -185,10 +185,14 @@ def create_parser(mach_interface=False):
             help="path to Node.js executable")
     add_arg('--browsertime-browsertimejs', dest='browsertime_browsertimejs',
             help="path to browsertime.js script")
+    add_arg('--browsertime-vismet-script', dest='browsertime_vismet_script',
+            help="path to visualmetrics.py script")
     add_arg('--browsertime-chromedriver', dest='browsertime_chromedriver',
             help="path to chromedriver executable")
     add_arg('--browsertime-video', dest='browsertime_video',
             help="records the viewport", default=False, action="store_true")
+    add_arg('--browsertime-visualmetrics', dest='browsertime_visualmetrics',
+            help="enables visual metrics", default=False, action="store_true")
     add_arg('--browsertime-no-ffwindowrecorder', dest='browsertime_no_ffwindowrecorder',
             help="disable the firefox window recorder", default=False, action="store_true")
     add_arg('--browsertime-ffmpeg', dest='browsertime_ffmpeg',
@@ -206,6 +210,10 @@ def verify_options(parser, args):
     if args.binary is None and args.app != "chrome-m":
         parser.error("--binary is required!")
 
+    # make sure that browsertime_video is set if visual metrics are requested
+    if args.browsertime_visualmetrics and not args.browsertime_video:
+        args.browsertime_video = True
+
     # if running chrome android tests, make sure it's on browsertime and
     # that the chromedriver path was provided
     if args.app == "chrome-m":

testing/raptor/raptor/raptor.py

@@ -95,8 +95,7 @@ def main(args=sys.argv[1:]):
             # peel off arguments that are specific to browsertime
             for key in outer_kwargs.keys():
                 if key.startswith("browsertime_"):
-                    value = outer_kwargs.pop(key)
-                    inner_kwargs[key] = value
+                    inner_kwargs[key] = outer_kwargs.get(key)
 
             if args.app == "firefox" or args.app in CHROMIUM_DISTROS:
                 klass = BrowsertimeDesktop

testing/raptor/raptor/results.py

@@ -283,6 +283,7 @@ class BrowsertimeResultsHandler(PerftestResultsHandler):
     def __init__(self, config, root_results_dir=None):
         super(BrowsertimeResultsHandler, self).__init__(**config)
         self._root_results_dir = root_results_dir
+        self.browsertime_visualmetrics = False
 
     def result_dir(self):
         return self._root_results_dir
@@ -502,6 +503,25 @@ class BrowsertimeResultsHandler(PerftestResultsHandler):
                 power_result["statistics"] = raw_result["statistics"]["android"]["power"]
                 results.append(power_result)
 
+            if self.browsertime_visualmetrics:
+                vismet_result = {
+                    "bt_ver": bt_ver,
+                    "browser": bt_browser,
+                    "url": bt_url,
+                    "measurements": {},
+                    "statistics": {},
+                }
+                for cycle in raw_result["visualMetrics"]:
+                    for metric in cycle:
+                        if "progress" in metric.lower():
+                            # Bug 1665750 - Determine if we should display progress
+                            continue
+                        vismet_result["measurements"].setdefault(
+                            metric, []
+                        ).append(cycle[metric])
+                vismet_result["statistics"] = raw_result["statistics"]["visualMetrics"]
+                results.append(vismet_result)
+
             custom_types = raw_result["browserScripts"][0].get("custom")
             if custom_types:
                 for custom_type in custom_types:

testing/raptor/test/conftest.py

@@ -24,6 +24,7 @@ def options(request):
         "app": "firefox",
         "binary": "path/to/dummy/browser",
         "no_conditioned_profile": True,
+        "browsertime_visualmetrics": False,
     }
 
     if hasattr(request.module, "OPTIONS"):
@@ -39,6 +40,7 @@ def browsertime_options(options):
     options["browsertime_geckodriver"] = "browsertime_geckodriver"
     options["browsertime_chromedriver"] = "browsertime_chromedriver"
     options["browsertime_video"] = "browsertime_video"
+    options["browsertime_visualmetrics"] = "browsertime_visualmetrics"
     options["browsertime_no_ffwindowrecorder"] = "browsertime_no_ffwindowrecorder"
     return options
 

testing/raptor/test/test_cmdline.py

@@ -29,6 +29,8 @@ def test_verify_options(filedir):
         memory_test=False,
         enable_webrender=False,
         chimera=False,
+        browsertime_video=False,
+        browsertime_visualmetrics=False,
     )
     parser = ArgumentParser()
 
@@ -51,6 +53,8 @@ def test_verify_options(filedir):
         memory_test=False,
         enable_webrender=False,
         chimera=False,
+        browsertime_video=False,
+        browsertime_visualmetrics=False,
     )
     verify_options(parser, args)  # assert no exception
 
@@ -67,6 +71,8 @@ def test_verify_options(filedir):
         memory_test=False,
         enable_webrender=False,
         chimera=False,
+        browsertime_video=False,
+        browsertime_visualmetrics=False,
     )
     verify_options(parser, args)  # assert no exception
 
@@ -83,6 +89,8 @@ def test_verify_options(filedir):
         memory_test=False,
         enable_webrender=False,
         chimera=False,
+        browsertime_video=False,
+        browsertime_visualmetrics=False,
     )
     verify_options(parser, args)  # assert no exception
 
@@ -99,6 +107,8 @@ def test_verify_options(filedir):
         memory_test=False,
         enable_webrender=False,
         chimera=False,
+        browsertime_video=False,
+        browsertime_visualmetrics=False,
     )
     verify_options(parser, args)  # assert no exception
 
@@ -115,6 +125,8 @@ def test_verify_options(filedir):
         memory_test=False,
         enable_webrender=False,
         chimera=False,
+        browsertime_video=False,
+        browsertime_visualmetrics=False,
     )
     parser = ArgumentParser()
 

tools/browsertime/mach_commands.py

@@ -404,7 +404,7 @@ class MachBrowsertime(MachCommandBase):
 
     def _need_install(self, package):
         from pip._internal.req.constructors import install_req_from_line
-        req = install_req_from_line("Pillow")
+        req = install_req_from_line(package)
         req.check_if_exists(use_user_site=False)
         if req.satisfied_by is None:
             return True