diff --git a/testing/mochitest/browser-test.js b/testing/mochitest/browser-test.js index e27c0ce908aa..e9e68a907121 100644 --- a/testing/mochitest/browser-test.js +++ b/testing/mochitest/browser-test.js @@ -180,6 +180,8 @@ function Tester(aTests, aDumper, aCallback) { this.SimpleTestOriginal[m] = this.SimpleTest[m]; }); + this._coverageCollector = null; + this._toleratedUncaughtRejections = null; this._uncaughtErrorObserver = function({message, date, fileName, stack, lineNumber}) { let error = message; @@ -247,6 +249,13 @@ Tester.prototype = { if (gConfig.repeat) this.repeat = gConfig.repeat; + if (gConfig.jscovDirPrefix) { + let coveragePath = gConfig.jscovDirPrefix; + let {CoverageCollector} = Cu.import("resource://testing-common/CoverageUtils.jsm", + {}); + this._coverageCollector = new CoverageCollector(coveragePath); + } + this.dumper.structuredLogger.info("*** Start BrowserChrome Test Results ***"); Services.console.registerListener(this); Services.obs.addObserver(this, "chrome-document-global-created", false); @@ -423,6 +432,9 @@ Tester.prototype = { nextTest: Task.async(function*() { if (this.currentTest) { this.Promise.Debugging.flushUncaughtErrors(); + if (this._coverageCollector) { + this._coverageCollector.recordTestCoverage(this.currentTest.path); + } // Run cleanup functions for the current test before moving on to the // next one. @@ -562,6 +574,9 @@ Tester.prototype = { // is invoked to start the tests. this.waitForWindowsState((function () { if (this.done) { + if (this._coverageCollector) { + this._coverageCollector.finalize(); + } // Uninitialize a few things explicitly so that they can clean up // frames and browser intentionally kept alive until shutdown to diff --git a/testing/mochitest/mochitest_options.py b/testing/mochitest/mochitest_options.py index 0048b9f6c77c..26ffd76dce93 100644 --- a/testing/mochitest/mochitest_options.py +++ b/testing/mochitest/mochitest_options.py @@ -401,6 +401,14 @@ class MochitestArguments(ArgumentContainer): "default": None, "suppress": True, }], + [["--jscov-dir-prefix"], + {"action": "store", + "help": "Directory to store per-test line coverage data as json " + "(browser-chrome only). To emit lcov formatted data, set " + "JS_CODE_COVERAGE_OUTPUT_DIR in the environment.", + "default": None, + "suppress": True, + }], [["--strict-content-sandbox"], {"action": "store_true", "default": False, @@ -683,6 +691,13 @@ class MochitestArguments(ArgumentContainer): "directory for %s does not exist as a destination to copy a " "chrome manifest." % options.store_chrome_manifest) + if options.jscov_dir_prefix: + options.jscov_dir_prefix = os.path.abspath(options.jscov_dir_prefix) + if not os.path.isdir(options.jscov_dir_prefix): + parser.error( + "directory %s does not exist as a destination for coverage " + "data." % options.jscov_dir_prefix) + if options.testingModulesDir is None: if build_obj: options.testingModulesDir = os.path.join( diff --git a/testing/mochitest/runtests.py b/testing/mochitest/runtests.py index 1bcabf4e8030..5ffd528649db 100644 --- a/testing/mochitest/runtests.py +++ b/testing/mochitest/runtests.py @@ -1158,6 +1158,8 @@ overlay chrome://browser/content/browser.xul chrome://mochikit/content/jetpack-a d = dict((k, v) for k, v in options.__dict__.items() if (v is None) or isinstance(v, (basestring, numbers.Number))) d['testRoot'] = self.testRoot + if options.jscov_dir_prefix: + d['jscovDirPrefix'] = options.jscov_dir_prefix; if not options.keep_open: d['closeWhenDone'] = '1' content = json.dumps(d) diff --git a/testing/modules/CoverageUtils.jsm b/testing/modules/CoverageUtils.jsm new file mode 100644 index 000000000000..74114c3da444 --- /dev/null +++ b/testing/modules/CoverageUtils.jsm @@ -0,0 +1,129 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +this.EXPORTED_SYMBOLS = [ + "CoverageCollector", +] + +const Cc = Components.classes; +const Ci = Components.interfaces; +const Cu = Components.utils; + +const {TextEncoder, OS} = Cu.import("resource://gre/modules/osfile.jsm", {}); +const {addDebuggerToGlobal} = Cu.import("resource://gre/modules/jsdebugger.jsm", + {}); +addDebuggerToGlobal(this); + +/** + * Records coverage for each test by way of the js debugger. + */ +this.CoverageCollector = function (prefix) { + this._prefix = prefix; + this._dbg = new Debugger(); + this._dbg.collectCoverageInfo = true; + this._dbg.addAllGlobalsAsDebuggees(); + this._scripts = this._dbg.findScripts(); + + this._dbg.onNewScript = (script) => { + this._scripts.push(script); + }; + + // Source -> coverage data; + this._allCoverage = {}; + this._encoder = new TextEncoder(); + this._testIndex = 0; +} + +CoverageCollector.prototype._getLinesCovered = function () { + let coveredLines = {}; + let currentCoverage = {}; + this._scripts.forEach(s => { + let scriptName = s.url; + let cov = s.getOffsetsCoverage(); + if (!cov) { + return; + } + + cov.forEach(covered => { + let {lineNumber, columnNumber, offset, count} = covered; + if (!count) { + return; + } + + if (!currentCoverage[scriptName]) { + currentCoverage[scriptName] = {}; + } + if (!this._allCoverage[scriptName]) { + this._allCoverage[scriptName] = {}; + } + + let key = [lineNumber, columnNumber, offset].join('#'); + if (!currentCoverage[scriptName][key]) { + currentCoverage[scriptName][key] = count; + } else { + currentCoverage[scriptName][key] += count; + } + }); + + }); + + // Covered lines are determined by comparing every offset mentioned as of the + // the completion of a test to the last time we measured coverage. If an + // offset in a line is novel as of this test, or a count has increased for + // any offset on a particular line, that line must have been covered. + for (let scriptName in currentCoverage) { + for (let key in currentCoverage[scriptName]) { + if (!this._allCoverage[scriptName] || + !this._allCoverage[scriptName][key] || + (this._allCoverage[scriptName][key] < + currentCoverage[scriptName][key])) { + let [lineNumber, colNumber, offset] = key.split('#'); + if (!coveredLines[scriptName]) { + coveredLines[scriptName] = new Set(); + } + coveredLines[scriptName].add(parseInt(lineNumber, 10)); + this._allCoverage[scriptName][key] = currentCoverage[scriptName][key]; + } + } + } + + return coveredLines; +} + + +/** + * Records lines covered since the last time coverage was recorded, + * associating them with the given test name. The result is written + * to a json file in a specified directory. + */ +CoverageCollector.prototype.recordTestCoverage = function (testName) { + dump("Collecting coverage for: " + testName + "\n"); + let rawLines = this._getLinesCovered(testName); + let result = []; + for (let scriptName in rawLines) { + let rec = { + testUrl: testName, + sourceFile: scriptName, + covered: [] + }; + for (let line of rawLines[scriptName]) { + rec.covered.push(line); + } + result.push(rec); + } + let arr = this._encoder.encode(JSON.stringify(result, null, 2)); + let path = this._prefix + '/' + 'jscov_' + Date.now() + '.json'; + dump("Writing coverage to: " + path + "\n"); + return OS.File.writeAtomic(path, arr, {tmpPath: path + '.tmp'}); +} + +/** + * Tear down the debugger after all tests are complete. + */ +CoverageCollector.prototype.finalize = function () { + this._dbg.removeAllDebuggees(); + this._dbg.enabled = false; +} diff --git a/testing/modules/moz.build b/testing/modules/moz.build index 82bf03362e57..7f7cc323b923 100644 --- a/testing/modules/moz.build +++ b/testing/modules/moz.build @@ -11,6 +11,7 @@ TESTING_JS_MODULES += [ 'AppData.jsm', 'AppInfo.jsm', 'Assert.jsm', + 'CoverageUtils.jsm', 'MockRegistrar.jsm', 'StructuredLog.jsm', 'TestUtils.jsm', diff --git a/testing/mozharness/configs/unittests/linux_unittest.py b/testing/mozharness/configs/unittests/linux_unittest.py index d4b75787cbdc..e431594d6c3b 100644 --- a/testing/mozharness/configs/unittests/linux_unittest.py +++ b/testing/mozharness/configs/unittests/linux_unittest.py @@ -187,6 +187,7 @@ config = { "browser-chrome": ["--browser-chrome"], "browser-chrome-chunked": ["--browser-chrome", "--chunk-by-runtime"], "browser-chrome-addons": ["--browser-chrome", "--chunk-by-runtime", "--tag=addons"], + "browser-chrome-coverage": ["--timeout=1200"], "mochitest-gl": ["--subsuite=webgl"], "mochitest-devtools-chrome": ["--browser-chrome", "--subsuite=devtools"], "mochitest-devtools-chrome-chunked": ["--browser-chrome", "--subsuite=devtools", "--chunk-by-runtime"], diff --git a/testing/mozharness/scripts/desktop_unittest.py b/testing/mozharness/scripts/desktop_unittest.py index d7d8369c92ca..ebb4dac55ba4 100755 --- a/testing/mozharness/scripts/desktop_unittest.py +++ b/testing/mozharness/scripts/desktop_unittest.py @@ -384,6 +384,10 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix if suite_category not in c["suite_definitions"]: self.fatal("'%s' not defined in the config!") + if suite == 'browser-chrome-coverage': + base_cmd.append('--jscov-dir-prefix=%s' % + dirs['abs_blob_upload_dir']) + options = c["suite_definitions"][suite_category]["options"] if options: for option in options: