Bug 1578862 - Feed --browsertime Raptor task videos into new visual metrics task type r=rwood,sparky,jlorenzo,barret

Feed --browsertime Raptor task videos into new visual metrics task type

Differential Revision: https://phabricator.services.mozilla.com/D48656

--HG--
extra : moz-landing-system : lando

taskcluster/ci/test/raptor.yml

@@ -1024,8 +1024,11 @@ browsertime-tp6-1:
             firefox: true
             default: false
     treeherder-symbol: Btime(tp6-1)
-    max-run-time: 3000
+    max-run-time: 4000
     tier: 3
+    attributes:
+        run-visual-metrics: true
     mozharness:
         extra-options:
             - --browsertime
+            - --browsertime-video

taskcluster/ci/visual-metrics-dep/kind.yml

@@ -0,0 +1,43 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: taskgraph.loader.single_dep:loader
+
+kind-dependencies:
+    - fetch
+    - test
+
+transforms:
+    - taskgraph.transforms.name_sanity:transforms
+    - taskgraph.transforms.visual_metrics_dep:transforms
+    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.task:transforms
+
+only-for-attributes:
+    - run-visual-metrics
+
+
+job-template:
+    description: "Run visual metrics calculations on Raptor"
+    run-on-projects: ['mozilla-central', 'try']
+    worker-type: t-linux-xlarge
+
+    treeherder:
+        tier: 3
+        kind: other
+
+    worker:
+        docker-image: {in-tree: visual-metrics}
+        max-run-time: 900
+        artifacts:
+            - type: file
+              name: public/visual-metrics.tar.xz
+              path: /builds/worker/artifacts/visual-metrics.tar.xz
+    fetches:
+        fetch:
+            - visual-metrics
+    run:
+        using: run-task
+        command: /builds/worker/bin/run-visual-metrics.py --jobs-json-path /builds/worker/fetches/jobs.json -- --orange --perceptual --contentful --force --renderignore 5 --json --viewport
+        checkout: false

taskcluster/ci/visual-metrics/kind.yml

@@ -31,7 +31,7 @@ jobs:
             artifacts:
                 - type: file
                   name: public/visual-metrics.tar.xz
-                  path: /builds/worker/visual-metrics.tar.xz
+                  path: /builds/worker/artifacts/visual-metrics.tar.xz
 
         fetches:
             fetch:

taskcluster/docker/visual-metrics/run-visual-metrics.py

@@ -64,15 +64,33 @@ class Job:
 JOB_SCHEMA = Schema(
     {
         Required("jobs"): [
-            {
-                Required("browsertime_json_url"): Url(),
-                Required("video_url"): Url(),
-            }
+            {Required("browsertime_json_url"): Url(), Required("video_url"): Url()}
         ]
     }
 )
 
 
+def run_command(log, cmd):
+    """Run a command using subprocess.check_output
+
+    Args:
+        log: The structlog logger instance.
+        cmd: the command to run as a list of strings.
+
+    Returns:
+        A tuple of the process' exit status and standard output.
+    """
+    log.info("Running command", cmd=cmd)
+    try:
+        res = subprocess.check_output(cmd)
+        log.info("Command succeeded", result=res)
+        return 0, res
+    except subprocess.CalledProcessError as e:
+        log.info("Command failed", cmd=cmd, status=e.returncode,
+                 output=e.output)
+        return e.returncode, e.output
+
+
 def main(log, args):
     """Run visualmetrics.py in parallel.
 
@@ -91,14 +109,14 @@ def main(log, args):
     visualmetrics_path = Path(fetch_dir) / "visualmetrics.py"
     if not visualmetrics_path.exists():
         log.error(
-            "Could not locate visualmetrics.py: expected it at %s"
-            % visualmetrics_path
+            "Could not locate visualmetrics.py",
+            expected_path=str(visualmetrics_path)
         )
         return 1
 
     if args.jobs_json_path:
         try:
-            with open(args.jobs_json_path, "r") as f:
+            with open(str(args.jobs_json_path), "r") as f:
                 jobs_json = json.load(f)
         except Exception as e:
             log.error(
@@ -109,9 +127,7 @@ def main(log, args):
             return 1
 
         log.info(
-            "Loaded jobs.json from file",
-            path=args.jobs_json_path,
-            jobs_json=jobs_json,
+            "Loaded jobs.json from file", path=args.jobs_json_path, jobs_json=jobs_json
         )
 
     else:
@@ -161,15 +177,16 @@ def main(log, args):
                 downloaded_jobs,
             ),
         ):
-            if isinstance(result, Exception):
+            returncode, res = result
+            if returncode != 0:
                 log.error(
-                    "Failed to run visualmetrics.py",
-                    video_url=job.video_url,
-                    error=result,
+                    "Failed to run visualmetrics.py", video_url=job.video_url, error=res
                 )
             else:
-                with (job.job_dir / "visual-metrics.json").open("wb") as f:
-                    f.write(result)
+                path = job.job_dir / "visual-metrics.json"
+                with path.open("wb") as f:
+                    log.info("Writing job result", path=path)
+                    f.write(res)
 
     OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
 
@@ -180,33 +197,25 @@ def main(log, args):
                     {
                         "video_url": job.video_url,
                         "browsertime_json_url": job.json_url,
-                        "path": (
-                            str(job.job_dir.relative_to(WORKSPACE_DIR)) + "/"
-                        ),
+                        "path": (str(job.job_dir.relative_to(WORKSPACE_DIR)) + "/"),
                     }
                     for job in downloaded_jobs
                 ],
                 "failed_jobs": [
-                    {
-                        "video_url": job.video_url,
-                        "browsertime_json_url": job.json_url,
-                    }
+                    {"video_url": job.video_url, "browsertime_json_url": job.json_url}
                     for job in failed_jobs
                 ],
             },
             f,
         )
 
-    subprocess.check_output(
-        [
-            "tar",
-            "cJf",
-            str(OUTPUT_DIR / "visual-metrics.tar.xz"),
-            "-C",
-            str(WORKSPACE_DIR),
-            ".",
-        ]
+    tarfile = OUTPUT_DIR / "visual-metrics.tar.xz"
+    log.info("Creating the tarfile", tarfile=tarfile)
+    returncode, res = run_command(
+        log, ["tar", "cJf", str(tarfile), "-C", str(WORKSPACE_DIR), "."]
     )
+    if returncode != 0:
+        raise Exception("Could not tar the results")
 
 
 def download_inputs(log, raw_jobs):
@@ -240,9 +249,7 @@ def download_inputs(log, raw_jobs):
     failed_jobs = []
 
     with ThreadPoolExecutor(max_workers=8) as executor:
-        for job, success in executor.map(
-            partial(download_job, log), pending_jobs
-        ):
+        for job, success in executor.map(partial(download_job, log), pending_jobs):
             if success:
                 downloaded_jobs.append(job)
             else:
@@ -330,22 +337,11 @@ def run_visual_metrics(job, visualmetrics_path, options):
     """Run visualmetrics.py on the input job.
 
     Returns:
-       Either a string containing the JSON output of visualmetrics.py or an
-       exception raised by :func:`subprocess.check_output`.
+       A returncode and a string containing the output of visualmetrics.py
     """
-    cmd = [
-        "/usr/bin/python",
-        str(visualmetrics_path),
-        "--video",
-        str(job.video_path),
-    ]
-
+    cmd = ["/usr/bin/python", str(visualmetrics_path), "--video", str(job.video_path)]
     cmd.extend(options)
-
-    try:
-        return subprocess.check_output(cmd)
-    except subprocess.CalledProcessError as e:
-        return e
+    return run_command(log, cmd)
 
 
 if __name__ == "__main__":
@@ -359,18 +355,17 @@ if __name__ == "__main__":
     )
 
     parser = argparse.ArgumentParser(
-        description=__doc__,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
     )
     parser.add_argument(
         "--jobs-json-path",
         type=Path,
         metavar="PATH",
         help=(
-            "The path to the jobs.josn file. If not present, the "
+            "The path to the jobs.json file. If not present, the "
             "VISUAL_METRICS_JOBS_JSON environment variable will be used "
             "instead."
-        ),
+        )
     )
     parser.add_argument(
         "visual_metrics_options",

taskcluster/docs/attributes.rst

@@ -326,3 +326,8 @@ disable-build-signing
 =====================
 Some GeckoView-only tasks produce APKs, but not APKs that should be
 signed.  Set this to ``true`` to disable APK signing.
+
+run-visual-metrics
+==================
+If set to true, will run the visual metrics task on the provided
+video files.

taskcluster/docs/kinds.rst

@@ -608,6 +608,11 @@ visual-metrics
 Tasks that compute visual performance metrics from videos and images captured
 by other tasks.
 
+visual-metrics-dep
+------------------
+Tasks that compute visual performance metrics from videos and images captured
+by another task that produces a jobs.json artifact
+
 iris
 ----
 Iris testing suite

taskcluster/taskgraph/transforms/visual_metrics_dep.py

@@ -0,0 +1,38 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+These transformations take a task description for a visual metrics task and
+add the necessary environment variables to run on the given inputs.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+from taskgraph.transforms.base import TransformSequence
+
+
+transforms = TransformSequence()
+
+SYMBOL = "%(groupSymbol)s(%(symbol)s-vismet)"
+LABEL = "vismet-%(platform)s-%(raptor_try_name)s"
+
+
+@transforms.add
+def run_visual_metrics(config, jobs):
+    for job in jobs:
+        dep_job = job.pop('primary-dependency', None)
+        if dep_job is not None:
+            platform = dep_job.task['extra']['treeherder-platform']
+            job['dependencies'] = {dep_job.label: dep_job.label}
+            job['fetches'][dep_job.label] = ['/public/test_info/jobs.json']
+            attributes = dict(dep_job.attributes)
+            attributes['platform'] = platform
+            job['label'] = LABEL % attributes
+            treeherder_info = dict(dep_job.task['extra']['treeherder'])
+            job['treeherder']['symbol'] = SYMBOL % treeherder_info
+
+            # vismet runs on Linux but we want to have it displayed
+            # alongside the job it was triggered by to make it easier for
+            # people to find it back.
+            job['treeherder']['platform'] = platform
+
+            yield job

testing/mozharness/mozharness/mozilla/testing/raptor.py

@@ -78,6 +78,12 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin):
             "default": None,
             "help": argparse.SUPPRESS
         }],
+        [["--browsertime-video"], {
+            "dest": "browsertime_video",
+            "action": "store_true",
+            "default": False,
+            "help": argparse.SUPPRESS
+        }],
         [["--browsertime"], {
             "dest": "browsertime",
             "action": "store_true",

testing/raptor/raptor/cmdline.py

@@ -157,6 +157,8 @@ def create_parser(mach_interface=False):
             help="path to browsertime.js script")
     add_arg('--browsertime-chromedriver', dest='browsertime_chromedriver',
             help="path to chromedriver executable")
+    add_arg('--browsertime-video', dest='browsertime_video',
+            help="records the viewport", default=False, action="store_true")
     add_arg('--browsertime-ffmpeg', dest='browsertime_ffmpeg',
             help="path to ffmpeg executable (for `--video=true`)")
     add_arg('--browsertime-geckodriver', dest='browsertime_geckodriver',

testing/raptor/raptor/raptor.py

@@ -238,6 +238,7 @@ either Raptor or browsertime."""
             raptor_json_path = os.path.join(os.getcwd(), 'local.json')
 
         self.config['raptor_json_path'] = raptor_json_path
+        self.config['artifact_dir'] = self.artifact_dir
         return self.results_handler.summarize_and_output(self.config, tests, test_names)
 
     @abstractmethod
@@ -356,6 +357,8 @@ class Browsertime(Perftest):
                   "browsertime_ffmpeg",
                   "browsertime_geckodriver",
                   "browsertime_chromedriver"):
+            if not self.browsertime_video and k == "browsertime_ffmpeg":
+                continue
             LOG.info("{}: {}".format(k, getattr(self, k)))
             try:
                 LOG.info("{}: {}".format(k, os.stat(getattr(self, k))))
@@ -477,7 +480,7 @@ class Browsertime(Perftest):
                browsertime_script +
                ['--firefox.profileTemplate', str(self.profile.profile),
                 '--skipHar',
-                '--video', 'false',
+                '--video', self.browsertime_video and 'true' or 'false',
                 '--visualMetrics', 'false',
                 '--timeouts.pageLoad', str(timeout),
                 '-vv',
@@ -487,12 +490,13 @@ class Browsertime(Perftest):
         LOG.info('timeout (s): {}'.format(timeout))
         LOG.info('browsertime cwd: {}'.format(os.getcwd()))
         LOG.info('browsertime cmd: {}'.format(cmd))
-        LOG.info('browsertime_ffmpeg: {}'.format(self.browsertime_ffmpeg))
+        if self.browsertime_video:
+            LOG.info('browsertime_ffmpeg: {}'.format(self.browsertime_ffmpeg))
 
         # browsertime requires ffmpeg on the PATH for `--video=true`.
         # It's easier to configure the PATH here than at the TC level.
         env = dict(os.environ)
-        if self.browsertime_ffmpeg:
+        if self.browsertime_video and self.browsertime_ffmpeg:
             ffmpeg_dir = os.path.dirname(os.path.abspath(self.browsertime_ffmpeg))
             old_path = env.setdefault('PATH', '')
             new_path = os.pathsep.join([ffmpeg_dir, old_path])

testing/raptor/raptor/results.py

@@ -340,6 +340,26 @@ class BrowsertimeResultsHandler(PerftestResultsHandler):
 
         return results
 
+    def _extract_vmetrics_jobs(self, test, browsertime_json, browsertime_results):
+        # XXX will do better later
+        url = ("https://queue.taskcluster.net/v1/task/%s/runs/0/artifacts/public/"
+               "test_info/" % os.environ.get("TASK_ID", "??"))
+
+        json_url = url + "/".join(browsertime_json.split(os.path.sep)[-3:])
+        files = []
+        for res in browsertime_results:
+            files.extend(res.get("files", {}).get("video", []))
+        if len(files) == 0:
+            # no video files.
+            return None
+        name = browsertime_json.split(os.path.sep)[-2]
+        result = []
+        for file in files:
+            video_url = url + "browsertime-results/" + name + "/" + file
+            result.append({"browsertime_json_url": json_url,
+                           "video_url": video_url})
+        return result
+
     def summarize_and_output(self, test_config, tests, test_names):
         """
         Retrieve, process, and output the browsertime test results. Currently supports page-load
@@ -366,6 +386,11 @@ class BrowsertimeResultsHandler(PerftestResultsHandler):
         # summarize the browsertime result data, write to file and output PERFHERDER_DATA
         LOG.info("retrieving browsertime test results")
 
+        # video_jobs is populated with video files produced by browsertime, we
+        # will send to the visual metrics task
+        video_jobs = []
+        run_local = test_config.get('run_local', False)
+
         for test in tests:
             bt_res_json = os.path.join(self.result_dir_for_test(test), 'browsertime.json')
             if os.path.exists(bt_res_json):
@@ -383,6 +408,11 @@ class BrowsertimeResultsHandler(PerftestResultsHandler):
                 LOG.error("Exception: %s %s" % (type(e).__name__, str(e)))
                 raise
 
+            if not run_local:
+                video_files = self._extract_vmetrics_jobs(test, bt_res_json, raw_btresults)
+                if video_files:
+                    video_jobs.extend(video_files)
+
             for new_result in self.parse_browsertime_json(raw_btresults):
                 # add additional info not from the browsertime json
                 for field in ('name', 'unit', 'lower_is_better',
@@ -416,4 +446,11 @@ class BrowsertimeResultsHandler(PerftestResultsHandler):
         if not self.gecko_profile:
             validate_success = self._validate_treeherder_data(output, out_perfdata)
 
+        # Dumping the video list for the visual metrics task.
+        if len(video_jobs) > 0:
+            jobs_file = os.path.join(test_config["artifact_dir"], "jobs.json")
+            LOG.info("Writing %d video jobs into %s" % (len(video_jobs), jobs_file))
+            with open(jobs_file, "w") as f:
+                f.write(json.dumps({"jobs": video_jobs}))
+
         return success and validate_success