From f70fad68df3834b71a62cd782f4e2e6e318a677d Mon Sep 17 00:00:00 2001 From: Joel Maher Date: Fri, 17 Oct 2014 10:01:15 -0400 Subject: [PATCH] Bug 987849 - Add fail-if support to Mochitest manifests. r=ahal. --- .../webspeech/synth/ipc/test/file_ipc.html | 1 + testing/mochitest/browser-harness.xul | 3 +- testing/mochitest/browser-test.js | 22 +++++++++++-- testing/mochitest/chrome/chrome.ini | 4 +++ .../mochitest/chrome/test_sanityManifest.xul | 19 +++++++++++ .../chrome/test_sanityManifest_pf.xul | 20 +++++++++++ testing/mochitest/chunkifyTests.js | 6 +++- testing/mochitest/manifestLibrary.js | 18 +++++++--- testing/mochitest/runtests.py | 9 +++-- testing/mochitest/server.js | 14 ++++---- .../tests/Harness_sanity/mochitest.ini | 7 ++++ .../Harness_sanity/test_sanity_manifest.html | 16 +++++++++ .../test_sanity_manifest_pf.html | 17 ++++++++++ .../mochitest/tests/SimpleTest/SimpleTest.js | 33 +++++++++++++++++-- .../mochitest/tests/SimpleTest/TestRunner.js | 17 ++++++++-- testing/mochitest/tests/SimpleTest/setup.js | 2 +- testing/mochitest/tests/browser/browser.ini | 30 ++++++++++------- testing/mochitest/tests/browser/moz.build | 8 ----- testing/mochitest/tests/moz.build | 5 ++- 19 files changed, 204 insertions(+), 47 deletions(-) create mode 100644 testing/mochitest/chrome/test_sanityManifest.xul create mode 100644 testing/mochitest/chrome/test_sanityManifest_pf.xul create mode 100644 testing/mochitest/tests/Harness_sanity/test_sanity_manifest.html create mode 100644 testing/mochitest/tests/Harness_sanity/test_sanity_manifest_pf.html delete mode 100644 testing/mochitest/tests/browser/moz.build diff --git a/content/media/webspeech/synth/ipc/test/file_ipc.html b/content/media/webspeech/synth/ipc/test/file_ipc.html index b18c8d0ac7d0..5f5a8ae0c415 100644 --- a/content/media/webspeech/synth/ipc/test/file_ipc.html +++ b/content/media/webspeech/synth/ipc/test/file_ipc.html @@ -13,6 +13,7 @@ window.SimpleTest = parent.SimpleTest; window.ok = parent.ok; window.is = parent.is; + window.info = parent.info; function iframeScriptFirst() { content.wrappedJSObject.RunSet.reloadAndRunAll({ diff --git a/testing/mochitest/browser-harness.xul b/testing/mochitest/browser-harness.xul index cce9d7173696..42443fb51a76 100644 --- a/testing/mochitest/browser-harness.xul +++ b/testing/mochitest/browser-harness.xul @@ -119,7 +119,8 @@ var gErrorCount = 0; function browserTest(aTestFile) { - this.path = aTestFile; + this.path = aTestFile['url']; + this.expected = aTestFile['expected']; this.dumper = gDumper; this.results = []; this.scope = null; diff --git a/testing/mochitest/browser-test.js b/testing/mochitest/browser-test.js index 2315e869789c..bcef56e9d293 100644 --- a/testing/mochitest/browser-test.js +++ b/testing/mochitest/browser-test.js @@ -357,6 +357,10 @@ Tester.prototype = { } }; + if (testScope.__expected == 'fail' && testScope.__num_failed <= 0) { + this.currentTest.addResult(new testResult(false, "We expected at least one assertion to fail because this test file was marked as fail-if in the manifest", "", false)); + } + this.Promise.Debugging.flushUncaughtErrors(); let winUtils = window.QueryInterface(Ci.nsIInterfaceRequestor) @@ -575,7 +579,7 @@ Tester.prototype = { this.SimpleTest.reset(); // Load the tests into a testscope - let currentScope = this.currentTest.scope = new testScope(this, this.currentTest); + let currentScope = this.currentTest.scope = new testScope(this, this.currentTest, this.currentTest.expected); let currentTest = this.currentTest; // Import utils in the test scope. @@ -804,11 +808,20 @@ function testMessage(aName) { // Need to be careful adding properties to this object, since its properties // cannot conflict with global variables used in tests. -function testScope(aTester, aTest) { +function testScope(aTester, aTest, expected) { this.__tester = aTester; + this.__expected = expected; + this.__num_failed = 0; var self = this; this.ok = function test_ok(condition, name, diag, stack) { + if (this.__expected == 'fail') { + if (!condition) { + this.__num_failed++; + condition = true; + } + } + aTest.addResult(new testResult(condition, name, diag, false, stack ? stack : Components.stack.caller)); }; @@ -930,6 +943,10 @@ function testScope(aTester, aTest) { self.__expectedMaxAsserts = max; }; + this.setExpected = function test_setExpected() { + self.__expected = 'fail'; + }; + this.finish = function test_finish() { self.__done = true; if (self.__waitTimer) { @@ -959,6 +976,7 @@ testScope.prototype = { __timeoutFactor: 1, __expectedMinAsserts: 0, __expectedMaxAsserts: 0, + __expected: 'pass', EventUtils: {}, SimpleTest: {}, diff --git a/testing/mochitest/chrome/chrome.ini b/testing/mochitest/chrome/chrome.ini index 5366e90d698b..ecd391111d9b 100644 --- a/testing/mochitest/chrome/chrome.ini +++ b/testing/mochitest/chrome/chrome.ini @@ -8,4 +8,8 @@ skip-if = buildapp == 'mulet' skip-if = buildapp == 'mulet' [test_sanityException.xul] [test_sanityException2.xul] +[test_sanityManifest.xul] +fail-if = true +[test_sanityManifest_pf.xul] +fail-if = true [test_chromeGetTestFile.xul] diff --git a/testing/mochitest/chrome/test_sanityManifest.xul b/testing/mochitest/chrome/test_sanityManifest.xul new file mode 100644 index 000000000000..ff92c37325fc --- /dev/null +++ b/testing/mochitest/chrome/test_sanityManifest.xul @@ -0,0 +1,19 @@ + + + + + + + + + + diff --git a/testing/mochitest/chrome/test_sanityManifest_pf.xul b/testing/mochitest/chrome/test_sanityManifest_pf.xul new file mode 100644 index 000000000000..7500d155e759 --- /dev/null +++ b/testing/mochitest/chrome/test_sanityManifest_pf.xul @@ -0,0 +1,20 @@ + + + + + + + + + + diff --git a/testing/mochitest/chunkifyTests.js b/testing/mochitest/chunkifyTests.js index 55d6756cfaef..0b1f23f67389 100644 --- a/testing/mochitest/chunkifyTests.js +++ b/testing/mochitest/chunkifyTests.js @@ -15,7 +15,11 @@ function chunkifyTests(tests, totalChunks, thisChunk, chunkByDir, logger) { var tests_by_dir = {}; var test_dirs = [] for (var i = 0; i < tests.length; ++i) { - var test_path = tests[i]; + if ('test' in tests[i]) { + var test_path = tests[i]['test']['url']; + } else { + var test_path = tests[i]['url']; + } if (test_path[0] == '/') { test_path = test_path.substr(1); } diff --git a/testing/mochitest/manifestLibrary.js b/testing/mochitest/manifestLibrary.js index 0f5f5ed39cb8..6858500d39ca 100644 --- a/testing/mochitest/manifestLibrary.js +++ b/testing/mochitest/manifestLibrary.js @@ -25,9 +25,11 @@ function parseTestManifest(testManifest, params, callback) { continue; } if (params.testRoot != 'tests' && params.testRoot !== undefined) { - links[params.baseurl + '/' + params.testRoot + '/' + path] = true + name = params.baseurl + '/' + params.testRoot + '/' + path; + links[name] = {'test': {'url': name, 'expected': obj['expected']}}; } else { - paths.push(params.testPrefix + path); + name = params.testPrefix + path; + paths.push({'test': {'url': name, 'expected': obj['expected']}}); } } if (paths.length > 0) { @@ -100,7 +102,11 @@ function filterTests(filter, testList, runOnly) { // filteredTests. if (Object.keys(runtests).length) { for (var i = 0; i < testList.length; i++) { - var testpath = testList[i]; + if ((testList[i] instanceof Object) && ('test' in testList[i])) { + var testpath = testList[i]['test']['url']; + } else { + var testpath = testList[i]; + } var tmppath = testpath.replace(/^\//, ''); for (var f in runtests) { // Remove leading /tests/ if exists @@ -127,7 +133,11 @@ function filterTests(filter, testList, runOnly) { var refilteredTests = []; for (var i = 0; i < filteredTests.length; i++) { var found = false; - var testpath = filteredTests[i]; + if ((filteredTests[i] instanceof Object) && ('test' in filteredTests[i])) { + var testpath = filteredTests[i]['test']['url']; + } else { + var testpath = filteredTests[i]; + } var tmppath = testpath.replace(/^\//, ''); for (var f in excludetests) { // Remove leading /tests/ if exists diff --git a/testing/mochitest/runtests.py b/testing/mochitest/runtests.py index d784b70c7889..54e6c047a4af 100644 --- a/testing/mochitest/runtests.py +++ b/testing/mochitest/runtests.py @@ -661,7 +661,6 @@ class MochitestUtilsMixin(object): tests = self.getActiveTests(options, disabled) paths = [] - for test in tests: if testsToFilter and (test['path'] not in testsToFilter): continue @@ -1597,8 +1596,6 @@ class Mochitest(MochitestUtilsMixin): paths = [] for test in tests: - if test.get('expected') == 'fail': - raise Exception('fail-if encountered for test: %s. There is no support for fail-if in Mochitests.' % test['name']) pathAbs = os.path.abspath(test['path']) assert pathAbs.startswith(self.testRootAbs) tp = pathAbs[len(self.testRootAbs):].replace('\\', '/').strip('/') @@ -1612,8 +1609,10 @@ class Mochitest(MochitestUtilsMixin): continue testob = {'path': tp} - if test.has_key('disabled'): + if 'disabled' in test: testob['disabled'] = test['disabled'] + if 'expected' in test: + testob['expected'] = test['expected'] paths.append(testob) def path_sort(ob1, ob2): @@ -1643,7 +1642,7 @@ class Mochitest(MochitestUtilsMixin): testsToRun = [] for test in tests: - if test.has_key('disabled'): + if 'disabled' in test: continue testsToRun.append(test['path']) diff --git a/testing/mochitest/server.js b/testing/mochitest/server.js index 025945a6c400..c2a2b62fa984 100644 --- a/testing/mochitest/server.js +++ b/testing/mochitest/server.js @@ -421,7 +421,7 @@ function list(requestPath, directory, recurse) count += childCount; } else { if (file.leafName.charAt(0) != '.') { - links[key] = true; + links[key] = {'test': {'url': key, 'expected': 'pass'}}; } } } @@ -493,13 +493,13 @@ function linksToTableRows(links, recursionLevel) { var response = ""; for (var [link, value] in links) { - var classVal = (!isTest(link) && !(value instanceof Object)) + var classVal = (!isTest(link) && ((value instanceof Object) && ('test' in value))) ? "non-test invisible" : ""; var spacer = "padding-left: " + (10 * recursionLevel) + "px"; - if (value instanceof Object) { + if ((value instanceof Object) && !('test' in value)) { response += TR({class: "dir", id: "tr-" + link }, TD({colspan: "3"}, " "), TD({style: spacer}, @@ -535,10 +535,10 @@ function linksToTableRows(links, recursionLevel) function arrayOfTestFiles(linkArray, fileArray, testPattern) { for (var [link, value] in Iterator(linkArray)) { - if (value instanceof Object) { + if ((value instanceof Object) && !('test' in value)) { arrayOfTestFiles(value, fileArray, testPattern); - } else if (isTest(link, testPattern)) { - fileArray.push(link) + } else if (isTest(link, testPattern) && (value instanceof Object)) { + fileArray.push(value['test']) } } } @@ -549,7 +549,7 @@ function jsonArrayOfTestFiles(links) { var testFiles = []; arrayOfTestFiles(links, testFiles); - testFiles = ['"' + file + '"' for each(file in testFiles)]; + testFiles = ['"' + file['url'] + '"' for each(file in testFiles)]; return "[" + testFiles.join(",\n") + "]"; } diff --git a/testing/mochitest/tests/Harness_sanity/mochitest.ini b/testing/mochitest/tests/Harness_sanity/mochitest.ini index d130e7e50ee3..462a50b45490 100644 --- a/testing/mochitest/tests/Harness_sanity/mochitest.ini +++ b/testing/mochitest/tests/Harness_sanity/mochitest.ini @@ -23,3 +23,10 @@ skip-if = toolkit == 'android' || e10s #No test app installed skip-if = toolkit == 'android' #bug 688052 [test_sanitySimpletest.html] skip-if = toolkit == 'android' #bug 688052 +[test_sanity_manifest.html] +skip-if = toolkit == 'android' # we use the old manifest style on android +fail-if = true +[test_sanity_manifest_pf.html] +skip-if = toolkit == 'android' # we use the old manifest style on android +fail-if = true + diff --git a/testing/mochitest/tests/Harness_sanity/test_sanity_manifest.html b/testing/mochitest/tests/Harness_sanity/test_sanity_manifest.html new file mode 100644 index 000000000000..d8c791b4e2ff --- /dev/null +++ b/testing/mochitest/tests/Harness_sanity/test_sanity_manifest.html @@ -0,0 +1,16 @@ + + + + + SimpleTest.expected = 'fail' test + + + + + + + diff --git a/testing/mochitest/tests/Harness_sanity/test_sanity_manifest_pf.html b/testing/mochitest/tests/Harness_sanity/test_sanity_manifest_pf.html new file mode 100644 index 000000000000..1f71fdd5a4fa --- /dev/null +++ b/testing/mochitest/tests/Harness_sanity/test_sanity_manifest_pf.html @@ -0,0 +1,17 @@ + + + + + SimpleTest.expected = 'fail' test + + + + + + + diff --git a/testing/mochitest/tests/SimpleTest/SimpleTest.js b/testing/mochitest/tests/SimpleTest/SimpleTest.js index 83925a34b795..3220c74a9b7b 100644 --- a/testing/mochitest/tests/SimpleTest/SimpleTest.js +++ b/testing/mochitest/tests/SimpleTest/SimpleTest.js @@ -238,14 +238,33 @@ SimpleTest.testPluginIsOOP = function () { SimpleTest._tests = []; SimpleTest._stopOnLoad = true; SimpleTest._cleanupFunctions = []; +SimpleTest.expected = 'pass'; +SimpleTest.num_failed = 0; + +SimpleTest.setExpected = function () { + if (parent.TestRunner) { + SimpleTest.expected = parent.TestRunner.expected; + } +} +SimpleTest.setExpected(); /** * Something like assert. **/ SimpleTest.ok = function (condition, name, diag) { + var test = {'result': !!condition, 'name': name, 'diag': diag}; - var successInfo = {status:"PASS", expected:"PASS", message:"TEST-PASS"}; - var failureInfo = {status:"FAIL", expected:"PASS", message:"TEST-UNEXPECTED-FAIL"}; + if (SimpleTest.expected == 'fail') { + if (!test.result) { + SimpleTest.num_failed++; + test.result = !test.result; + } + var successInfo = {status:"PASS", expected:"PASS", message:"TEST-PASS"}; + var failureInfo = {status:"FAIL", expected:"FAIL", message:"TEST-KNOWN-FAIL"}; + } else { + var successInfo = {status:"PASS", expected:"PASS", message:"TEST-PASS"}; + var failureInfo = {status:"FAIL", expected:"PASS", message:"TEST-UNEXPECTED-FAIL"}; + } SimpleTest._logResult(test, successInfo, failureInfo); SimpleTest._tests.push(test); }; @@ -821,6 +840,16 @@ SimpleTest.finish = function() { } } + if (SimpleTest.expected == 'fail' && SimpleTest.num_failed <= 0) { + msg = 'We expected at least one failure'; + var test = {'result': false, 'name': 'fail-if condition in manifest', 'diag': msg}; + var successInfo = {status:"PASS", expected:"PASS", message:"TEST-PASS"}; + var failureInfo = {status:"FAIL", expected:"FAIL", message:"TEST-KNOWN-FAIL"}; + + SimpleTest._logResult(test, successInfo, failureInfo); + SimpleTest._tests.push(test); + } + SimpleTest.testsLength = SimpleTest._tests.length; SimpleTest._alreadyFinished = true; diff --git a/testing/mochitest/tests/SimpleTest/TestRunner.js b/testing/mochitest/tests/SimpleTest/TestRunner.js index 3bb580cdfe00..a3beca061dad 100644 --- a/testing/mochitest/tests/SimpleTest/TestRunner.js +++ b/testing/mochitest/tests/SimpleTest/TestRunner.js @@ -562,6 +562,19 @@ TestRunner.resetTests = function(listURLs) { TestRunner.runNextTest(); } +TestRunner.getNextUrl = function() { + var url = ""; + // sometimes we have a subtest/harness which doesn't use a manifest + if ((TestRunner._urls[TestRunner._currentTest] instanceof Object) && ('test' in TestRunner._urls[TestRunner._currentTest])) { + url = TestRunner._urls[TestRunner._currentTest]['test']['url']; + TestRunner.expected = TestRunner._urls[TestRunner._currentTest]['test']['expected']; + } else { + url = TestRunner._urls[TestRunner._currentTest]; + TestRunner.expected = 'pass'; + } + return url; +} + /** * Run the next test. If no test remains, calls onComplete(). **/ @@ -570,7 +583,7 @@ TestRunner.runNextTest = function() { if (TestRunner._currentTest < TestRunner._urls.length && !TestRunner._haltTests) { - var url = TestRunner._urls[TestRunner._currentTest]; + var url = TestRunner.getNextUrl(); TestRunner.currentTestURL = url; $("current-test-path").innerHTML = url; @@ -775,7 +788,7 @@ TestRunner.testUnloaded = function() { var numAsserts = newAssertionCount - TestRunner._lastAssertionCount; TestRunner._lastAssertionCount = newAssertionCount; - var url = TestRunner._urls[TestRunner._currentTest]; + var url = TestRunner.getNextUrl(); var max = TestRunner._expectedMaxAsserts; var min = TestRunner._expectedMinAsserts; if (numAsserts > max) { diff --git a/testing/mochitest/tests/SimpleTest/setup.js b/testing/mochitest/tests/SimpleTest/setup.js index 4657ad7c3144..9b2aca8622bc 100644 --- a/testing/mochitest/tests/SimpleTest/setup.js +++ b/testing/mochitest/tests/SimpleTest/setup.js @@ -241,7 +241,7 @@ function hookupTests(testList) { } else { gTestList = []; for (var obj in testList) { - gTestList.push(obj); + gTestList.push(testList[obj]); } } diff --git a/testing/mochitest/tests/browser/browser.ini b/testing/mochitest/tests/browser/browser.ini index 16e16891e1f7..9472e7921607 100644 --- a/testing/mochitest/tests/browser/browser.ini +++ b/testing/mochitest/tests/browser/browser.ini @@ -17,16 +17,24 @@ skip-if = e10s support-files = test-dir/* -# Disabled, these are only good for testing the harness' failure reporting -# browser_zz_fail_openwindow.js -# browser_fail.js -# browser_fail_add_task.js -# browser_fail_async_throw.js -# browser_fail_fp.js -# browser_fail_pf.js -# browser_fail_throw.js -# browser_fail_timeout.js -# browser_fail_unexpectedTimeout.js -# # Disabled because it would take too long, useful to check functionality though. # browser_requestLongerTimeout.js +[browser_zz_fail_openwindow.js] +skip-if = true # this catches outside of the main loop to find an extra window +[browser_fail.js] +skip-if = true +[browser_fail_add_task.js] +skip-if = true # fail-if doesnt catch an exception outside the test +[browser_fail_async_throw.js] +skip-if = true # fail-if doesnt catch an exception outside the test +[browser_fail_fp.js] +fail-if = true +[browser_fail_pf.js] +fail-if = true +[browser_fail_throw.js] +skip-if = true # fail-if doesnt catch an exception outside the test +[browser_fail_timeout.js] +fail-if = true +[browser_fail_unexpectedTimeout.js] +fail-if = false + diff --git a/testing/mochitest/tests/browser/moz.build b/testing/mochitest/tests/browser/moz.build deleted file mode 100644 index 33f04f853787..000000000000 --- a/testing/mochitest/tests/browser/moz.build +++ /dev/null @@ -1,8 +0,0 @@ -# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- -# vim: set filetype=python: -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -BROWSER_CHROME_MANIFESTS += ['browser.ini'] - diff --git a/testing/mochitest/tests/moz.build b/testing/mochitest/tests/moz.build index 130fa8a98c18..5126b84a80c7 100644 --- a/testing/mochitest/tests/moz.build +++ b/testing/mochitest/tests/moz.build @@ -5,9 +5,8 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. DIRS += [ - 'SimpleTest', - 'browser', + 'SimpleTest' ] MOCHITEST_MANIFESTS += ['Harness_sanity/mochitest.ini'] - +BROWSER_CHROME_MANIFESTS += ['browser/browser.ini']