Bug 1734381 - Add a tag to browsertime interactive tests. r=perftest-reviewers,AlexandruIonescu

Differential Revision: https://phabricator.services.mozilla.com/D127693

taskcluster/docker/visual-metrics/run-visual-metrics.py

@@ -43,6 +43,9 @@ class Job:
     #: A unique number for the job.
     count = attr.ib(type=int)
 
+    #: The tags for this job.
+    tags = attr.ib(type=str)
+
     #: The extra options for this job.
     extra_options = attr.ib(type=str)
 
@@ -63,6 +66,7 @@ JOB_SCHEMA = Schema(
             {
                 Required("test_name"): str,
                 Required("browsertime_json_path"): str,
+                Required("tags"): [str],
                 Required("extra_options"): [str],
                 Required("accept_zero_vismet"): bool,
             }
@@ -149,7 +153,7 @@ def run_command(log, cmd, job_count):
     return rc, res
 
 
-def append_result(log, suites, test_name, name, result, extra_options):
+def append_result(log, suites, test_name, name, result, tags, extra_options):
     """Appends a ``name`` metrics result in the ``test_name`` suite.
 
     Args:
@@ -177,7 +181,7 @@ def append_result(log, suites, test_name, name, result, extra_options):
         test_name,
         {
             "name": orig_test_name,
-            "tags": extra_options + ["visual"],
+            "tags": extra_options + tags + ["visual"],
             "subtests": {},
             "extraOptions": extra_options,
         },
@@ -326,6 +330,7 @@ def main(log, args):
                 jobs.append(
                     Job(
                         test_name=name,
+                        tags=job["tags"],
                         extra_options=len(job["extra_options"]) > 0
                         and job["extra_options"]
                         or jobs_json["extra_options"],
@@ -362,7 +367,13 @@ def main(log, args):
             else:
                 for name, value in res.items():
                     append_result(
-                        log, suites, job.test_name, name, value, job.extra_options
+                        log,
+                        suites,
+                        job.test_name,
+                        name,
+                        value,
+                        job.tags,
+                        job.extra_options,
                     )
 
     suites = [get_suite(suite) for suite in suites.values()]

testing/raptor/raptor/output.py

@@ -1513,7 +1513,9 @@ class BrowsertimeOutput(PerftestOutput):
                     "name": test["name"],
                     "type": test["type"],
                     "extraOptions": extra_options,
-                    "tags": test.get("tags", extra_options),
+                    # There may be unique options in tags now, but we don't want to remove the
+                    # previous behaviour which includes the extra options in the tags.
+                    "tags": test.get("tags", []) + extra_options,
                     "lowerIsBetter": test["lower_is_better"],
                     "unit": test["unit"],
                     "alertThreshold": float(test["alert_threshold"]),

testing/raptor/raptor/results.py

@@ -337,6 +337,18 @@ class BrowsertimeResultsHandler(PerftestResultsHandler):
         # not using control server with bt
         pass
 
+    def build_tags(self, test={}):
+        """Use to build the tags option for our perfherder data.
+
+        This should only contain items that will only be shown within
+        the tags section and excluded from the extra options.
+        """
+        tags = []
+        LOG.info(test)
+        if test.get("interactive", False):
+            tags.append("interactive")
+        return tags
+
     def parse_browsertime_json(
         self,
         raw_btresults,
@@ -637,6 +649,7 @@ class BrowsertimeResultsHandler(PerftestResultsHandler):
         test_name,
         browsertime_json,
         json_name="browsertime.json",
+        tags=[],
         extra_options=[],
         accept_zero_vismet=False,
     ):
@@ -651,6 +664,7 @@ class BrowsertimeResultsHandler(PerftestResultsHandler):
         return {
             "browsertime_json_path": _normalized_join(reldir, json_name),
             "test_name": test_name,
+            "tags": tags,
             "extra_options": extra_options,
             "accept_zero_vismet": accept_zero_vismet,
         }
@@ -727,6 +741,7 @@ class BrowsertimeResultsHandler(PerftestResultsHandler):
 
             if not run_local:
                 extra_options = self.build_extra_options()
+                tags = self.build_tags(test=test)
 
                 if self.chimera:
                     if cold_path is None or warm_path is None:
@@ -737,6 +752,7 @@ class BrowsertimeResultsHandler(PerftestResultsHandler):
                             test_name,
                             cold_path,
                             json_name="cold-browsertime.json",
+                            tags=list(tags),
                             extra_options=list(extra_options),
                             accept_zero_vismet=accept_zero_vismet,
                         )
@@ -749,6 +765,7 @@ class BrowsertimeResultsHandler(PerftestResultsHandler):
                             test_name,
                             warm_path,
                             json_name="warm-browsertime.json",
+                            tags=list(tags),
                             extra_options=list(extra_options),
                             accept_zero_vismet=accept_zero_vismet,
                         )
@@ -758,6 +775,7 @@ class BrowsertimeResultsHandler(PerftestResultsHandler):
                         self._extract_vmetrics(
                             test_name,
                             bt_res_json,
+                            tags=list(tags),
                             extra_options=list(extra_options),
                             accept_zero_vismet=accept_zero_vismet,
                         )
@@ -793,6 +811,7 @@ class BrowsertimeResultsHandler(PerftestResultsHandler):
                     new_result["subtest_unit"] = subtest_unit
 
                     new_result["extra_options"] = self.build_extra_options()
+                    new_result.setdefault("tags", []).extend(self.build_tags(test=test))
 
                     # Split the chimera
                     if self.chimera and "run=2" in new_result["url"][0]: