Bug 1278649 - Add code coverage to xpcshell tests. r=chmanchester

This adds the ability to use the command line flag '--jscov-dir-prefix' to collect javascript code coverage from xpcshell tests and output it into the specified directory as a JSON file.

MozReview-Commit-ID: 3MZm73SNChL

--HG--
extra : transplant_source : c%9B%DE%A93w%E7%11%89%BE-%E8%D9%18%BC%12z%0A%0E%E4

testing/mozharness/configs/unittests/linux_unittest.py

@@ -276,6 +276,11 @@ config = {
                         "--manifest=tests/xpcshell/tests/xpcshell.ini"],
             "tests": []
         },
+        "xpcshell-coverage": {
+            "options": ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
+                        "--manifest=tests/xpcshell/tests/xpcshell.ini"],
+            "tests": []
+        },
     },
     "all_cppunittest_suites": {
         "cppunittest": {"tests": ["tests/cppunittest"]}

testing/mozharness/scripts/desktop_unittest.py

@@ -386,7 +386,7 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix
             if suite_category not in c["suite_definitions"]:
                 self.fatal("'%s' not defined in the config!")
 
-            if suite == 'browser-chrome-coverage':
+            if suite in ('browser-chrome-coverage', 'xpcshell-coverage'):
                 base_cmd.append('--jscov-dir-prefix=%s' %
                                 dirs['abs_blob_upload_dir'])
 

testing/xpcshell/head.js

@@ -502,6 +502,12 @@ function _execute_test() {
   _PromiseTestUtils.init();
   _PromiseTestUtils.Assert = Assert;
 
+  let coverageCollector = null;
+  if (typeof _JSCOV_DIR === 'string') {
+    let _CoverageCollector = Components.utils.import("resource://testing-common/CoverageUtils.jsm", {}).CoverageCollector;
+    coverageCollector = new _CoverageCollector(_JSCOV_DIR);
+  }
+
   // _HEAD_FILES is dynamically defined by <runxpcshelltests.py>.
   _load_files(_HEAD_FILES);
   // _TEST_FILE is dynamically defined by <runxpcshelltests.py>.
@@ -529,6 +535,11 @@ function _execute_test() {
     } else {
       run_next_test();
     }
+
+    if (coverageCollector != null) {
+      coverageCollector.recordTestCoverage(_TEST_FILE);
+    }
+
     do_test_finished("MAIN run_test");
     _do_main();
     _PromiseTestUtils.assertNoUncaughtRejections();
@@ -539,6 +550,10 @@ function _execute_test() {
     // has already been logged so there is no need to log it again. It's
     // possible that this will mask an NS_ERROR_ABORT that happens after a
     // do_check failure though.
+    if (coverageCollector != null) {
+      coverageCollector.recordTestCoverage(_TEST_FILE);
+    }
+
     if (!_quit || e != Components.results.NS_ERROR_ABORT) {
       let extra = {};
       if (e.fileName) {
@@ -557,6 +572,10 @@ function _execute_test() {
     }
   }
 
+  if (coverageCollector != null) {
+    coverageCollector.finalize();
+  }
+
   // _TAIL_FILES is dynamically defined by <runxpcshelltests.py>.
   _load_files(_TAIL_FILES);
 
@@ -1249,6 +1268,10 @@ function do_load_child_test_harness()
       + "const _JSDEBUGGER_PORT=0; "
       + "const _XPCSHELL_PROCESS='child';";
 
+  if (typeof _JSCOV_DIR === 'string') {
+    command += " const _JSCOV_DIR=" + uneval(_JSCOV_DIR) + ";";
+  }
+
   if (_TESTING_MODULES_DIR) {
     command += " const _TESTING_MODULES_DIR=" + uneval(_TESTING_MODULES_DIR) + ";";
   }

testing/xpcshell/runxpcshelltests.py

@@ -132,6 +132,7 @@ class XPCShellTestThread(Thread):
         self.xpcshell = kwargs.get('xpcshell')
         self.xpcsRunArgs = kwargs.get('xpcsRunArgs')
         self.failureManifest = kwargs.get('failureManifest')
+        self.jscovdir = kwargs.get('jscovdir')
         self.stack_fixer_function = kwargs.get('stack_fixer_function')
         self._rootTempDir = kwargs.get('tempDir')
 
@@ -631,7 +632,14 @@ class XPCShellTestThread(Thread):
 
         # The test name to log
         cmdI = ['-e', 'const _TEST_NAME = "%s"' % name]
-        self.complete_command = cmdH + cmdT + cmdI + args
+
+        # Directory for javascript code coverage output, null by default.
+        cmdC = ['-e', 'const _JSCOV_DIR = null']
+        if self.jscovdir:
+            cmdC = ['-e', 'const _JSCOV_DIR = "%s"' % self.jscovdir.replace('\\', '/')]
+            self.complete_command = cmdH + cmdT + cmdI + cmdC + args
+        else:
+            self.complete_command = cmdH + cmdT + cmdI + args
 
         if self.test_object.get('dmd') == 'true':
             if sys.platform.startswith('linux'):
@@ -1081,7 +1089,7 @@ class XPCShellTests(object):
                  testClass=XPCShellTestThread, failureManifest=None,
                  log=None, stream=None, jsDebugger=False, jsDebuggerPort=0,
                  test_tags=None, dump_tests=None, utility_path=None,
-                 rerun_failures=False, failure_manifest=None, **otherOptions):
+                 rerun_failures=False, failure_manifest=None, jscovdir=None, **otherOptions):
         """Run xpcshell tests.
 
         |xpcshell|, is the xpcshell executable to use to run the tests.
@@ -1181,6 +1189,7 @@ class XPCShellTests(object):
         self.pluginsPath = pluginsPath
         self.sequential = sequential
         self.failure_manifest = failure_manifest
+        self.jscovdir = jscovdir
 
         self.testCount = 0
         self.passCount = 0
@@ -1263,6 +1272,7 @@ class XPCShellTests(object):
             'xpcshell': self.xpcshell,
             'xpcsRunArgs': self.xpcsRunArgs,
             'failureManifest': self.failure_manifest,
+            'jscovdir': self.jscovdir,
             'harness_timeout': self.harness_timeout,
             'stack_fixer_function': self.stack_fixer_function,
         }

testing/xpcshell/xpcshellcommandline.py

@@ -64,6 +64,10 @@ def add_common_arguments(parser):
                         action="store", type=str, dest="symbolsPath",
                         default=None,
                         help="absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
+    parser.add_argument("--jscov-dir-prefix",
+                        action="store", type=str, dest="jscovdir",
+                        default=argparse.SUPPRESS,
+                        help="Directory to store per-test javascript line coverage data as json.")
     parser.add_argument("--debugger",
                         action="store", dest="debugger",
                         help="use the given debugger to launch the application")