Bug 1325148 - Temporarily disable mochitest leakchecking for directories containing known leaks, r=jmaher

Leak checking in mochitest was failing to turn the jobs orange for a period of time. Several leaks slipped
through in the interim. In order to land the fix ASAP to avoid further bustage, these leaks are being
whitelisted. See the patch or the dependency graph of bug 1325148 for the list of leaks/bugs that are being
disabled.

While these bugs won't show up in treeherder, they are a priority to fix, as they could represent leaks that
are already landed in the code.

MozReview-Commit-ID: JY7Gpvswp4r

--HG--
extra : rebase_source : 0d75d80eccddca31a7d2d1c997a85efdb5a4858c

browser/extensions/shield-recipe-client/test/browser.ini

@@ -3,3 +3,4 @@
 [browser_EventEmitter.js]
 [browser_Storage.js]
 [browser_Heartbeat.js]
+skip-if = true # bug 1325409

testing/mochitest/runtests.py

@@ -824,6 +824,7 @@ class MochitestDesktop(object):
         self.result = {}
 
         self.start_script = os.path.join(here, 'start_desktop.js')
+        self.disable_leak_checking = False
 
     def update_mozinfo(self):
         """walk up directories to find mozinfo.json update the info"""
@@ -1495,7 +1496,8 @@ toolbar#nav-bar {
             self.log.error(str(e))
             return None
 
-        browserEnv["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
+        if not self.disable_leak_checking:
+            browserEnv["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
 
         try:
             gmp_path = self.getGMPPluginPath(options)
@@ -1939,12 +1941,13 @@ toolbar#nav-bar {
             args.append('-foreground')
             self.start_script_args.append(testUrl or 'about:blank')
 
-            if detectShutdownLeaks:
+            if detectShutdownLeaks and not self.disable_leak_checking:
                 shutdownLeaks = ShutdownLeaks(self.log)
             else:
                 shutdownLeaks = None
 
-            if mozinfo.info["asan"] and (mozinfo.isLinux or mozinfo.isMac):
+            if mozinfo.info["asan"] and (mozinfo.isLinux or mozinfo.isMac) \
+                    and not self.disable_leak_checking:
                 lsanLeaks = LSANLeaks(self.log)
             else:
                 lsanLeaks = None
@@ -2201,6 +2204,36 @@ toolbar#nav-bar {
         result = 1  # default value, if no tests are run.
         for d in dirs:
             print "dir: %s" % d
+
+            # BEGIN LEAKCHECK HACK
+            # Leak checking was broken in mochitest unnoticed for a length of time. During
+            # this time, several leaks slipped through. The leak checking was fixed by bug
+            # 1325148, but it couldn't land until all the regressions were also fixed or
+            # backed out. Rather than waiting and risking new regressions, in the meantime
+            # this code will selectively disable leak checking on flavors/directories where
+            # known regressions exist. At least this way we can prevent further damage while
+            # they get fixed.
+
+            info = mozinfo.info
+            skip_leak_conditions = [
+                (options.flavor in ('browser', 'chrome', 'plain') and d.startswith('toolkit/components/extensions/test/mochitest'), 'bug 1325158'),  # noqa
+                (info['debug'] and options.flavor == 'browser' and d.startswith('browser/components/extensions/test/browser'), 'bug 1325141'),  # noqa
+                (info['debug'] and options.flavor == 'plain' and d == 'dom/animation/test/css-animations', 'bug 1325277'),  # noqa
+                (info['debug'] and options.flavor == 'plain' and d == 'dom/tests/mochitest/gamepad' and info['os'] == 'win', 'bug 1324592'),  # noqa
+                (info['debug'] and options.flavor == 'plain' and d == 'toolkit/components/prompts/test' and info['os'] == 'mac', 'bug 1325275'),  # noqa
+                (info['debug'] and options.flavor == 'plain' and d == 'tests/dom/xhr/tests', 'bug 1325438'),  # noqa
+            ]
+
+            for condition, reason in skip_leak_conditions:
+                if condition:
+                    self.log.warning('WARNING | disabling leakcheck due to {}'.format(reason))
+                    self.disable_leak_checking = True
+                    break
+            else:
+                self.disable_leak_checking = False
+
+            # END LEAKCHECK HACK
+
             tests_in_dir = [t for t in testsToRun if os.path.dirname(t) == d]
 
             # If we are using --run-by-dir, we should not use the profile path (if) provided