Bug 1352357 - Add leak checking support for web-platform-tests, r=automatedtester

Implement gecko leak checking for web-platform-tests, but do not
enable it yet due to oranges. Allows disabling leak checks for a
specific test using a 'leaks' key in test metadata. This causes the
browser to restart before running the (group of) tests with leak
checking disabled. The feature can be enabled by passing --leak-check
on the command line.

MozReview-Commit-ID: 1kBnJkOaeu8

--HG--
extra : rebase_source : 2ad24889743a9a76a5027d68c46223617abd0f72

testing/web-platform/README.md

@@ -170,6 +170,19 @@ comma-seperate list of `pref.name:value` items to set e.g.
                 dom.serviceWorkers.exemptFromPerDomainMax:true,
                 dom.caches.enabled:true]
 
+Disabling Leak Checks
+----------------------
+
+When a test is imported that leaks, it may be necessary to temporarily
+disable leak checking for that test in order to allow the import to
+proceed. This works in basically the same way as disabling a test, but
+with the key 'leaks' e.g.
+
+    [filename.html]
+        type: testharness
+        leaks:
+            if os == "linux": https://bugzilla.mozilla.org/show_bug.cgi?id=1234567
+
 Setting per-Directory Metadata
 ------------------------------
 

testing/web-platform/harness/requirements_firefox.txt

@@ -3,3 +3,4 @@ mozprofile >= 0.21
 mozprocess >= 0.19
 mozcrash >= 0.13
 mozrunner >= 6.7
+mozleak >= 0.1

testing/web-platform/harness/wptrunner/browsers/base.py

@@ -87,8 +87,11 @@ class Browser(object):
         """Used for browser-specific setup that happens at the start of a test run"""
         pass
 
+    def settings(self, test):
+        return {}
+
     @abstractmethod
-    def start(self):
+    def start(self, **kwargs):
         """Launch the browser object and get it into a state where is is ready to run tests"""
         pass
 
@@ -130,7 +133,7 @@ class NullBrowser(Browser):
     def __init__(self, logger, **kwargs):
         super(NullBrowser, self).__init__(logger)
 
-    def start(self):
+    def start(self, **kwargs):
         """No-op browser to use in scenarios where the TestRunnerManager shouldn't
         actually own the browser process (e.g. Servo where we start one browser
         per test)"""

testing/web-platform/harness/wptrunner/browsers/chrome.py

@@ -67,7 +67,7 @@ class ChromeBrowser(Browser):
         self.binary = binary
         self.server = ChromeDriverServer(self.logger, binary=webdriver_binary)
 
-    def start(self):
+    def start(self, **kwargs):
         self.server.start(block=False)
 
     def stop(self, force=False):

testing/web-platform/harness/wptrunner/browsers/edge.py

@@ -48,7 +48,7 @@ class EdgeBrowser(Browser):
         self.webdriver_host = "localhost"
         self.webdriver_port = self.server.port
 
-    def start(self):
+    def start(self, **kwargs):
         print self.server.url
         self.server.start()
 

testing/web-platform/harness/wptrunner/browsers/firefox.py

@@ -9,6 +9,7 @@ import subprocess
 import sys
 
 import mozinfo
+import mozleak
 from mozprocess import ProcessHandler
 from mozprofile import FirefoxProfile, Preferences
 from mozprofile.permissions import ServerLocations
@@ -77,7 +78,8 @@ def browser_kwargs(test_type, run_info_data, **kwargs):
             "binary_args": kwargs["binary_args"],
             "timeout_multiplier": get_timeout_multiplier(test_type,
                                                          run_info_data,
-                                                         **kwargs)}
+                                                         **kwargs),
+            "leak_check": kwargs["leak_check"]}
 
 
 def executor_kwargs(test_type, server_config, cache_manager, run_info_data,
@@ -127,7 +129,7 @@ class FirefoxBrowser(Browser):
     def __init__(self, logger, binary, prefs_root, extra_prefs=None, debug_info=None,
                  symbols_path=None, stackwalk_binary=None, certutil_binary=None,
                  ca_certificate_path=None, e10s=False, stackfix_dir=None,
-                 binary_args=None, timeout_multiplier=None):
+                 binary_args=None, timeout_multiplier=None, leak_check=False):
         Browser.__init__(self, logger)
         self.binary = binary
         self.prefs_root = prefs_root
@@ -147,12 +149,20 @@ class FirefoxBrowser(Browser):
                                                         self.symbols_path)
         else:
             self.stack_fixer = None
+
         if timeout_multiplier:
             self.init_timeout = self.init_timeout * timeout_multiplier
 
-    def start(self):
-        self.marionette_port = get_free_port(2828, exclude=self.used_ports)
-        self.used_ports.add(self.marionette_port)
+        self.leak_report_file = None
+        self.leak_check = leak_check
+
+    def settings(self, test):
+        return {"check_leaks": self.leak_check and not test.leaks}
+
+    def start(self, **kwargs):
+        if self.marionette_port is None:
+            self.marionette_port = get_free_port(2828, exclude=self.used_ports)
+            self.used_ports.add(self.marionette_port)
 
         env = os.environ.copy()
         env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
@@ -172,6 +182,14 @@ class FirefoxBrowser(Browser):
         if self.e10s:
             self.profile.set_preferences({"browser.tabs.remote.autostart": True})
 
+        if self.leak_check and kwargs.get("check_leaks", True):
+            self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks.log")
+            if os.path.exists(self.leak_report_file):
+                os.remove(self.leak_report_file)
+            env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
+        else:
+            self.leak_report_file = None
+
         # Bug 1262954: winxp + e10s, disable hwaccel
         if (self.e10s and platform.system() in ("Windows", "Microsoft") and
             '5.1' in platform.version()):
@@ -227,6 +245,24 @@ class FirefoxBrowser(Browser):
             except OSError:
                 # This can happen on Windows if the process is already dead
                 pass
+        self.logger.debug("stopped")
+
+    def process_leaks(self):
+        self.logger.debug("PROCESS LEAKS %s" % self.leak_report_file)
+        if self.leak_report_file is None:
+            return
+        mozleak.process_leak_log(
+            self.leak_report_file,
+            leak_thresholds={
+                "default": 0,
+                "tab": 10000,  # See dependencies of bug 1051230.
+                # GMP rarely gets a log, but when it does, it leaks a little.
+                "geckomediaplugin": 20000,
+            },
+            ignore_missing_leaks=["geckomediaplugin"],
+            log=self.logger,
+            stack_fixer=self.stack_fixer
+        )
 
     def pid(self):
         if self.runner.process_handler is None:
@@ -253,6 +289,7 @@ class FirefoxBrowser(Browser):
 
     def cleanup(self):
         self.stop()
+        self.process_leaks()
 
     def executor_browser(self):
         assert self.marionette_port is not None

testing/web-platform/harness/wptrunner/browsers/servodriver.py

@@ -90,7 +90,7 @@ class ServoWebDriverBrowser(Browser):
         self.command = None
         self.user_stylesheets = user_stylesheets if user_stylesheets else []
 
-    def start(self):
+    def start(self, **kwargs):
         self.webdriver_port = get_free_port(4444, exclude=self.used_ports)
         self.used_ports.add(self.webdriver_port)
 

testing/web-platform/harness/wptrunner/manifestexpected.py

@@ -115,6 +115,10 @@ class ExpectedManifest(ManifestItem):
     def restart_after(self):
         return bool_prop("restart-after", self)
 
+    @property
+    def leaks(self):
+        return bool_prop("leaks", self)
+
     @property
     def tags(self):
         return tags(self)
@@ -133,6 +137,10 @@ class DirectoryManifest(ManifestItem):
     def restart_after(self):
         return bool_prop("restart-after", self)
 
+    @property
+    def leaks(self):
+        return bool_prop("leaks", self)
+
     @property
     def tags(self):
         return tags(self)
@@ -178,6 +186,10 @@ class TestNode(ManifestItem):
     def restart_after(self):
         return bool_prop("restart-after", self)
 
+    @property
+    def leaks(self):
+        return bool_prop("leaks", self)
+
     @property
     def tags(self):
         return tags(self)

testing/web-platform/harness/wptrunner/testrunner.py

@@ -157,11 +157,20 @@ class BrowserManager(object):
         self.logger = logger
         self.browser = browser
         self.no_timeout = no_timeout
+        self.browser_settings = None
 
         self.started = False
 
         self.init_timer = None
 
+    def update_settings(self, test):
+        browser_settings = self.browser.settings(test)
+        restart_required = ((self.browser_settings is not None and
+                             self.browser_settings != browser_settings) or
+                            test.expected() == "CRASH")
+        self.browser_settings = browser_settings
+        return restart_required
+
     def init(self):
         """Launch the browser that is being tested,
         and the TestRunner process that will run the tests."""
@@ -182,7 +191,8 @@ class BrowserManager(object):
             try:
                 if self.init_timer is not None:
                     self.init_timer.start()
-                self.browser.start()
+                self.logger.debug("Starting browser with settings %r" % self.browser_settings)
+                self.browser.start(**self.browser_settings)
                 self.browser_pid = self.browser.pid()
             except:
                 self.logger.warning("Failure during init %s" % traceback.format_exc())
@@ -441,6 +451,8 @@ class TestRunnerManager(threading.Thread):
             self.logger.error("Max restarts exceeded")
             return RunnerManagerState.error()
 
+        self.browser.update_settings(self.state.test)
+
         result = self.browser.init()
         if result is Stop:
             return RunnerManagerState.error()
@@ -508,6 +520,11 @@ class TestRunnerManager(threading.Thread):
         assert isinstance(self.state, RunnerManagerState.running)
         assert self.state.test is not None
 
+        if self.browser.update_settings(self.state.test):
+            self.logger.info("Restarting browser for new test environment")
+            return RunnerManagerState.restarting(self.state.test,
+                                                 self.state.test_queue)
+
         self.logger.test_start(self.state.test.id)
         self.send_message("run_test", self.state.test)
 

testing/web-platform/harness/wptrunner/wptcommandline.py

@@ -178,6 +178,8 @@ scheme host and port.""")
     gecko_group.add_argument("--setpref", dest="extra_prefs", action='append',
                              default=[], metavar="PREF=VALUE",
                              help="Defines an extra user preference (overrides those in prefs_root)")
+    gecko_group.add_argument("--leak-check", dest="leak_check", action="store_true",
+                             help="Enable leak checking")
 
     servo_group = parser.add_argument_group("Servo-specific")
     servo_group.add_argument("--user-stylesheet",

testing/web-platform/harness/wptrunner/wpttest.py

@@ -172,6 +172,14 @@ class Test(object):
                 return True
         return False
 
+    @property
+    def leaks(self):
+        for meta in self.itermeta(None):
+            leaks = meta.leaks
+            if leaks is not None:
+                return leaks
+        return False
+
     @property
     def tags(self):
         tags = set()