Bug 1631354 - very frequent browsertime speedometer logs too big to parse r=perftest-reviewers,sparky

Browsertime's -vvv displays the gecko profile passed through HTTP
in the logs, making them very big. This patch makes verbose output
an option and deactivates it by default.

Differential Revision: https://phabricator.services.mozilla.com/D71511

testing/mozharness/mozharness/mozilla/testing/raptor.py

@@ -292,6 +292,12 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin):
             "default": False,
             "help": "Enable cold page-load for browsertime tp6",
         }],
+        [["--verbose"], {
+            "action": "store_true",
+            "dest": "verbose",
+            "default": False,
+            "help": "Verbose output",
+        }],
 
     ] + testing_config_options + \
         copy.deepcopy(code_coverage_config_options) + \

testing/raptor/raptor/browsertime/base.py

@@ -205,10 +205,12 @@ class Browsertime(Perftest):
             "--timeouts.pageLoad", str(timeout),
             # running browser scripts timeout (milliseconds)
             "--timeouts.script", str(timeout * int(test.get("page_cycles", 1))),
-            "-vvv",
             "--resultDir", self.results_handler.result_dir_for_test(test),
         ]
 
+        if self.verbose:
+            browsertime_options.append("-vvv")
+
         if self.browsertime_video:
             # For now, capturing video with Firefox always uses the window recorder/composition
             # recorder.  In the future we'd like to be able to selectively use Android's `adb

testing/raptor/raptor/cmdline.py

@@ -189,7 +189,8 @@ def create_parser(mach_interface=False):
             help="path to ffmpeg executable (for `--video=true`)")
     add_arg('--browsertime-geckodriver', dest='browsertime_geckodriver',
             help="path to geckodriver executable")
-
+    add_arg('--verbose', dest="verbose", action="store_true", default=False,
+            help="Verbose output")
     add_logging_group(parser)
     return parser
 

testing/raptor/raptor/perftest.py

@@ -97,9 +97,11 @@ either Raptor or browsertime."""
         conditioned_profile_scenario='settled',
         extra_prefs={},
         project="mozilla-central",
+        verbose=False,
         **kwargs
     ):
         self._dirs_to_remove = []
+        self.verbose = verbose
 
         # Override the magic --host HOST_IP with the value of the environment variable.
         if host == "HOST_IP":
@@ -130,7 +132,8 @@ either Raptor or browsertime."""
             "disable_perf_tuning": disable_perf_tuning,
             "conditioned_profile_scenario": conditioned_profile_scenario,
             "extra_prefs": extra_prefs,
-            "project": project
+            "project": project,
+            "verbose": verbose
         }
 
         self.firefox_android_apps = FIREFOX_ANDROID_APPS

testing/raptor/raptor/raptor.py

@@ -134,7 +134,8 @@ def main(args=sys.argv[1:]):
             no_conditioned_profile=args.no_conditioned_profile,
             disable_perf_tuning=args.disable_perf_tuning,
             conditioned_profile_scenario=args.conditioned_profile_scenario,
-            project=args.project
+            project=args.project,
+            verbose=args.verbose
         )
     except Exception:
         traceback.print_exc()

testing/raptor/test/test_raptor.py

@@ -4,6 +4,7 @@ import os
 import sys
 import threading
 import time
+import traceback
 import mock
 import pytest
 from mock import Mock
@@ -38,6 +39,12 @@ class TestBrowserThread(threading.Thread):
         self.names = names
         self.exc = None
 
+    def print_error(self):
+        if self.exc is None:
+            return
+        type, value, tb = self.exc
+        traceback.print_exception(type, value, tb, None, sys.stderr)
+
     def run(self):
         try:
             self.raptor_instance.run_tests(self.tests, self.names)
@@ -207,7 +214,10 @@ def test_start_browser(get_binary, app):
         except RunnerNotStartedError:
             time.sleep(0.1)
     else:
-        assert False  # browser didn't start
+        # browser didn't start
+        # if the thread had an error, display it here
+        thread.print_error()
+        assert False
 
     raptor.clean_up()
     thread.join(5)
@@ -236,7 +246,6 @@ def test_cmd_arguments(ConcreteBrowsertime, browsertime_options, mock_test):
         "--visualMetrics", "false",
         "--timeouts.pageLoad", str(DEFAULT_TIMEOUT),
         "--timeouts.script", str(DEFAULT_TIMEOUT),
-        "-vvv",
         "--resultDir",
         "-n", "1",
     }