Bug 987849 - Add fail-if support to Mochitest manifests. r=ahal.

2024-11-26 14:22:01 +00:00 · 2014-10-17 10:01:15 -04:00 · 2014-10-17 10:01:15 -04:00 · f70fad68df
commit f70fad68df
parent 8f259e6613
19 changed files with 204 additions and 47 deletions
--- a/content/media/webspeech/synth/ipc/test/file_ipc.html
+++ b/content/media/webspeech/synth/ipc/test/file_ipc.html
@ -13,6 +13,7 @@
    window.SimpleTest = parent.SimpleTest;
    window.ok = parent.ok;
    window.is = parent.is;
+    window.info = parent.info;

    function iframeScriptFirst() {
      content.wrappedJSObject.RunSet.reloadAndRunAll({
--- a/testing/mochitest/browser-harness.xul
+++ b/testing/mochitest/browser-harness.xul
@ -119,7 +119,8 @@
    var gErrorCount = 0;

    function browserTest(aTestFile) {
-      this.path = aTestFile;
+      this.path = aTestFile['url'];
+      this.expected = aTestFile['expected'];
      this.dumper = gDumper;
      this.results = [];
      this.scope = null;
--- a/testing/mochitest/browser-test.js
+++ b/testing/mochitest/browser-test.js
@ -357,6 +357,10 @@ Tester.prototype = {
        }
      };

+      if (testScope.__expected == 'fail' && testScope.__num_failed <= 0) {
+        this.currentTest.addResult(new testResult(false, "We expected at least one assertion to fail because this test file was marked as fail-if in the manifest", "", false));
+      }
+
      this.Promise.Debugging.flushUncaughtErrors();

      let winUtils = window.QueryInterface(Ci.nsIInterfaceRequestor)
@ -575,7 +579,7 @@ Tester.prototype = {
    this.SimpleTest.reset();

    // Load the tests into a testscope
-    let currentScope = this.currentTest.scope = new testScope(this, this.currentTest);
+    let currentScope = this.currentTest.scope = new testScope(this, this.currentTest, this.currentTest.expected);
    let currentTest = this.currentTest;

    // Import utils in the test scope.
@ -804,11 +808,20 @@ function testMessage(aName) {

 // Need to be careful adding properties to this object, since its properties
 // cannot conflict with global variables used in tests.
-function testScope(aTester, aTest) {
+function testScope(aTester, aTest, expected) {
  this.__tester = aTester;
+  this.__expected = expected;
+  this.__num_failed = 0;

  var self = this;
  this.ok = function test_ok(condition, name, diag, stack) {
+    if (this.__expected == 'fail') {
+        if (!condition) {
+          this.__num_failed++;
+          condition = true;
+        }
+    }
+
    aTest.addResult(new testResult(condition, name, diag, false,
                                   stack ? stack : Components.stack.caller));
  };
@ -930,6 +943,10 @@ function testScope(aTester, aTest) {
    self.__expectedMaxAsserts = max;
  };

+  this.setExpected = function test_setExpected() {
+    self.__expected = 'fail';
+  };
+
  this.finish = function test_finish() {
    self.__done = true;
    if (self.__waitTimer) {
@ -959,6 +976,7 @@ testScope.prototype = {
  __timeoutFactor: 1,
  __expectedMinAsserts: 0,
  __expectedMaxAsserts: 0,
+  __expected: 'pass',

  EventUtils: {},
  SimpleTest: {},
--- a/testing/mochitest/chrome/chrome.ini
+++ b/testing/mochitest/chrome/chrome.ini
@ -8,4 +8,8 @@ skip-if = buildapp == 'mulet'
 skip-if = buildapp == 'mulet'
 [test_sanityException.xul]
 [test_sanityException2.xul]
+[test_sanityManifest.xul]
+fail-if = true
+[test_sanityManifest_pf.xul]
+fail-if = true
 [test_chromeGetTestFile.xul]
--- a/testing/mochitest/chrome/test_sanityManifest.xul
+++ b/testing/mochitest/chrome/test_sanityManifest.xul
@ -0,0 +1,19 @@
+<?xml version="1.0"?>
+<?xml-stylesheet href="chrome://global/skin" type="text/css"?>
+<?xml-stylesheet href="chrome://mochikit/content/tests/SimpleTest/test.css" type="text/css"?>
+<!--
+https://bugzilla.mozilla.org/show_bug.cgi?id=670817
+-->
+<window title="Mozilla Bug 987849"
+  xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul">
+
+  <script type="application/javascript" src="chrome://mochikit/content/tests/SimpleTest/SimpleTest.js"/>
+
+<body xmlns="http://www.w3.org/1999/xhtml">
+<a target="_blank" href="https://bugzilla.mozilla.org/show_bug.cgi?id=987849">Mozilla Bug 987849</a>
+<script type="application/javascript"><![CDATA[
+ok(false, "a call to ok");
+]]></script>
+</body>
+
+</window>
--- a/testing/mochitest/chrome/test_sanityManifest_pf.xul
+++ b/testing/mochitest/chrome/test_sanityManifest_pf.xul
@ -0,0 +1,20 @@
+<?xml version="1.0"?>
+<?xml-stylesheet href="chrome://global/skin" type="text/css"?>
+<?xml-stylesheet href="chrome://mochikit/content/tests/SimpleTest/test.css" type="text/css"?>
+<!--
+https://bugzilla.mozilla.org/show_bug.cgi?id=670817
+-->
+<window title="Mozilla Bug 987849"
+  xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul">
+
+  <script type="application/javascript" src="chrome://mochikit/content/tests/SimpleTest/SimpleTest.js"/>
+
+<body xmlns="http://www.w3.org/1999/xhtml">
+<a target="_blank" href="https://bugzilla.mozilla.org/show_bug.cgi?id=987849">Mozilla Bug 987849</a>
+<script type="application/javascript"><![CDATA[
+ok(true, "a true call to ok");
+ok(false, "a false call to ok");
+]]></script> 
+</body>
+
+</window>
--- a/testing/mochitest/chunkifyTests.js
+++ b/testing/mochitest/chunkifyTests.js
@ -15,7 +15,11 @@ function chunkifyTests(tests, totalChunks, thisChunk, chunkByDir, logger) {
    var tests_by_dir = {};
    var test_dirs = []
    for (var i = 0; i < tests.length; ++i) {
-      var test_path = tests[i];
+      if ('test' in tests[i]) {
+        var test_path = tests[i]['test']['url'];
+      } else {
+        var test_path = tests[i]['url'];
+      }
      if (test_path[0] == '/') {
        test_path = test_path.substr(1);
      }
--- a/testing/mochitest/manifestLibrary.js
+++ b/testing/mochitest/manifestLibrary.js
@ -25,9 +25,11 @@ function parseTestManifest(testManifest, params, callback) {
      continue;
    }
    if (params.testRoot != 'tests' && params.testRoot !== undefined) {
-      links[params.baseurl + '/' + params.testRoot + '/' + path] = true
+      name = params.baseurl + '/' + params.testRoot + '/' + path;
+      links[name] = {'test': {'url': name, 'expected': obj['expected']}};
    } else {
-      paths.push(params.testPrefix + path);
+      name = params.testPrefix + path;
+      paths.push({'test': {'url': name, 'expected': obj['expected']}});
    }
  }
  if (paths.length > 0) {
@ -100,7 +102,11 @@ function filterTests(filter, testList, runOnly) {
  // filteredTests.
  if (Object.keys(runtests).length) {
    for (var i = 0; i < testList.length; i++) {
+      if ((testList[i] instanceof Object) && ('test' in testList[i])) {
+        var testpath = testList[i]['test']['url'];
+      } else {
        var testpath = testList[i];
+      }
      var tmppath = testpath.replace(/^\//, '');
      for (var f in runtests) {
        // Remove leading /tests/ if exists
@ -127,7 +133,11 @@ function filterTests(filter, testList, runOnly) {
  var refilteredTests = [];
  for (var i = 0; i < filteredTests.length; i++) {
    var found = false;
+    if ((filteredTests[i] instanceof Object) && ('test' in filteredTests[i])) {
+      var testpath = filteredTests[i]['test']['url'];
+    } else {
      var testpath = filteredTests[i];
+    }
    var tmppath = testpath.replace(/^\//, '');
    for (var f in excludetests) {
      // Remove leading /tests/ if exists
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@ -661,7 +661,6 @@ class MochitestUtilsMixin(object):

    tests = self.getActiveTests(options, disabled)
    paths = []
-
    for test in tests:
      if testsToFilter and (test['path'] not in testsToFilter):
        continue
@ -1597,8 +1596,6 @@ class Mochitest(MochitestUtilsMixin):
    paths = []

    for test in tests:
-      if test.get('expected') == 'fail':
-        raise Exception('fail-if encountered for test: %s. There is no support for fail-if in Mochitests.' % test['name'])
      pathAbs = os.path.abspath(test['path'])
      assert pathAbs.startswith(self.testRootAbs)
      tp = pathAbs[len(self.testRootAbs):].replace('\\', '/').strip('/')
@ -1612,8 +1609,10 @@ class Mochitest(MochitestUtilsMixin):
        continue

      testob = {'path': tp}
-      if test.has_key('disabled'):
+      if 'disabled' in test:
        testob['disabled'] = test['disabled']
+      if 'expected' in test:
+        testob['expected'] = test['expected']
      paths.append(testob)

    def path_sort(ob1, ob2):
@ -1643,7 +1642,7 @@ class Mochitest(MochitestUtilsMixin):

    testsToRun = []
    for test in tests:
-      if test.has_key('disabled'):
+      if 'disabled' in test:
        continue
      testsToRun.append(test['path'])

--- a/testing/mochitest/server.js
+++ b/testing/mochitest/server.js
@ -421,7 +421,7 @@ function list(requestPath, directory, recurse)
      count += childCount;
    } else {
      if (file.leafName.charAt(0) != '.') {
-        links[key] = true;
+        links[key] = {'test': {'url': key, 'expected': 'pass'}};
      }
    }
  }
@ -493,13 +493,13 @@ function linksToTableRows(links, recursionLevel)
 {
  var response = "";
  for (var [link, value] in links) {
-    var classVal = (!isTest(link) && !(value instanceof Object))
+    var classVal = (!isTest(link) && ((value instanceof Object) && ('test' in value)))
      ? "non-test invisible"
      : "";

    var spacer = "padding-left: " + (10 * recursionLevel) + "px";

-    if (value instanceof Object) {
+    if ((value instanceof Object) && !('test' in value)) {
      response += TR({class: "dir", id: "tr-" + link },
                     TD({colspan: "3"}, "&#160;"),
                     TD({style: spacer},
@ -535,10 +535,10 @@ function linksToTableRows(links, recursionLevel)

 function arrayOfTestFiles(linkArray, fileArray, testPattern) {
  for (var [link, value] in Iterator(linkArray)) {
-    if (value instanceof Object) {
+    if ((value instanceof Object) && !('test' in value)) {
      arrayOfTestFiles(value, fileArray, testPattern);
-    } else if (isTest(link, testPattern)) {
-      fileArray.push(link)
+    } else if (isTest(link, testPattern) && (value instanceof Object)) {
+      fileArray.push(value['test'])
    }
  }
 }
@ -549,7 +549,7 @@ function jsonArrayOfTestFiles(links)
 {
  var testFiles = [];
  arrayOfTestFiles(links, testFiles);
-  testFiles = ['"' + file + '"' for each(file in testFiles)];
+  testFiles = ['"' + file['url'] + '"' for each(file in testFiles)];
  return "[" + testFiles.join(",\n") + "]";
 }

--- a/testing/mochitest/tests/Harness_sanity/mochitest.ini
+++ b/testing/mochitest/tests/Harness_sanity/mochitest.ini
@ -23,3 +23,10 @@ skip-if = toolkit == 'android' || e10s #No test app installed
 skip-if = toolkit == 'android' #bug 688052
 [test_sanitySimpletest.html]
 skip-if = toolkit == 'android' #bug 688052
+[test_sanity_manifest.html]
+skip-if = toolkit == 'android' # we use the old manifest style on android
+fail-if = true
+[test_sanity_manifest_pf.html]
+skip-if = toolkit == 'android' # we use the old manifest style on android
+fail-if = true
+
--- a/testing/mochitest/tests/Harness_sanity/test_sanity_manifest.html
+++ b/testing/mochitest/tests/Harness_sanity/test_sanity_manifest.html
@ -0,0 +1,16 @@
+<!-- Any copyright is dedicated to the Public Domain.
+   - http://creativecommons.org/publicdomain/zero/1.0/ -->
+<!DOCTYPE HTML>
+<html>
+<head>
+  <title>SimpleTest.expected = 'fail' test</title>
+  <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" />
+</head>
+<body>
+<script class="testbody" type="text/javascript">
+ok(false, "We expect this to fail");
+
+</script>
+</body>
+</html>
--- a/testing/mochitest/tests/Harness_sanity/test_sanity_manifest_pf.html
+++ b/testing/mochitest/tests/Harness_sanity/test_sanity_manifest_pf.html
@ -0,0 +1,17 @@
+<!-- Any copyright is dedicated to the Public Domain.
+   - http://creativecommons.org/publicdomain/zero/1.0/ -->
+<!DOCTYPE HTML>
+<html>
+<head>
+  <title>SimpleTest.expected = 'fail' test</title>
+  <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" />
+</head>
+<body>
+<script class="testbody" type="text/javascript">
+ok(true, "We expect this to pass");
+ok(false, "We expect this to fail");
+
+</script>
+</body>
+</html>
--- a/testing/mochitest/tests/SimpleTest/SimpleTest.js
+++ b/testing/mochitest/tests/SimpleTest/SimpleTest.js
@ -238,14 +238,33 @@ SimpleTest.testPluginIsOOP = function () {
 SimpleTest._tests = [];
 SimpleTest._stopOnLoad = true;
 SimpleTest._cleanupFunctions = [];
+SimpleTest.expected = 'pass';
+SimpleTest.num_failed = 0;
+
+SimpleTest.setExpected = function () {
+  if (parent.TestRunner) {
+    SimpleTest.expected = parent.TestRunner.expected;
+  }
+}
+SimpleTest.setExpected();

 /**
 * Something like assert.
 **/
 SimpleTest.ok = function (condition, name, diag) {
+
    var test = {'result': !!condition, 'name': name, 'diag': diag};
+    if (SimpleTest.expected == 'fail') {
+      if (!test.result) {
+        SimpleTest.num_failed++;
+        test.result = !test.result;
+      }
+      var successInfo = {status:"PASS", expected:"PASS", message:"TEST-PASS"};
+      var failureInfo = {status:"FAIL", expected:"FAIL", message:"TEST-KNOWN-FAIL"};
+    } else {
      var successInfo = {status:"PASS", expected:"PASS", message:"TEST-PASS"};
      var failureInfo = {status:"FAIL", expected:"PASS", message:"TEST-UNEXPECTED-FAIL"};
+    }
    SimpleTest._logResult(test, successInfo, failureInfo);
    SimpleTest._tests.push(test);
 };
@ -821,6 +840,16 @@ SimpleTest.finish = function() {
        }
    }

+    if (SimpleTest.expected == 'fail' && SimpleTest.num_failed <= 0) {
+        msg = 'We expected at least one failure';
+        var test = {'result': false, 'name': 'fail-if condition in manifest', 'diag': msg};
+        var successInfo = {status:"PASS", expected:"PASS", message:"TEST-PASS"};
+        var failureInfo = {status:"FAIL", expected:"FAIL", message:"TEST-KNOWN-FAIL"};
+
+        SimpleTest._logResult(test, successInfo, failureInfo);
+        SimpleTest._tests.push(test);
+    }
+
    SimpleTest.testsLength = SimpleTest._tests.length;

    SimpleTest._alreadyFinished = true;
--- a/testing/mochitest/tests/SimpleTest/TestRunner.js
+++ b/testing/mochitest/tests/SimpleTest/TestRunner.js
@ -562,6 +562,19 @@ TestRunner.resetTests = function(listURLs) {
  TestRunner.runNextTest();
 }

+TestRunner.getNextUrl = function() {
+    var url = "";
+    // sometimes we have a subtest/harness which doesn't use a manifest
+    if ((TestRunner._urls[TestRunner._currentTest] instanceof Object) && ('test' in TestRunner._urls[TestRunner._currentTest])) {
+        url = TestRunner._urls[TestRunner._currentTest]['test']['url'];
+        TestRunner.expected = TestRunner._urls[TestRunner._currentTest]['test']['expected'];
+    } else {
+        url = TestRunner._urls[TestRunner._currentTest];
+        TestRunner.expected = 'pass';
+    }
+    return url;
+}
+
 /**
 * Run the next test. If no test remains, calls onComplete().
 **/
@ -570,7 +583,7 @@ TestRunner.runNextTest = function() {
    if (TestRunner._currentTest < TestRunner._urls.length &&
        !TestRunner._haltTests)
    {
-        var url = TestRunner._urls[TestRunner._currentTest];
+        var url = TestRunner.getNextUrl();
        TestRunner.currentTestURL = url;

        $("current-test-path").innerHTML = url;
@ -775,7 +788,7 @@ TestRunner.testUnloaded = function() {
        var numAsserts = newAssertionCount - TestRunner._lastAssertionCount;
        TestRunner._lastAssertionCount = newAssertionCount;

-        var url = TestRunner._urls[TestRunner._currentTest];
+        var url = TestRunner.getNextUrl();
        var max = TestRunner._expectedMaxAsserts;
        var min = TestRunner._expectedMinAsserts;
        if (numAsserts > max) {
--- a/testing/mochitest/tests/SimpleTest/setup.js
+++ b/testing/mochitest/tests/SimpleTest/setup.js
@ -241,7 +241,7 @@ function hookupTests(testList) {
  } else {
    gTestList = [];
    for (var obj in testList) {
-        gTestList.push(obj);
+        gTestList.push(testList[obj]);
    }
  }

--- a/testing/mochitest/tests/browser/browser.ini
+++ b/testing/mochitest/tests/browser/browser.ini
@ -17,16 +17,24 @@ skip-if = e10s
 support-files =
  test-dir/*

-# Disabled, these are only good for testing the harness' failure reporting
-#  browser_zz_fail_openwindow.js
-#  browser_fail.js
-#  browser_fail_add_task.js
-#  browser_fail_async_throw.js
-#  browser_fail_fp.js
-#  browser_fail_pf.js
-#  browser_fail_throw.js
-#  browser_fail_timeout.js
-#  browser_fail_unexpectedTimeout.js
-#
 # Disabled because it would take too long, useful to check functionality though.
 #  browser_requestLongerTimeout.js
+[browser_zz_fail_openwindow.js]
+skip-if = true # this catches outside of the main loop to find an extra window
+[browser_fail.js]
+skip-if = true
+[browser_fail_add_task.js]
+skip-if = true # fail-if doesnt catch an exception outside the test
+[browser_fail_async_throw.js]
+skip-if = true # fail-if doesnt catch an exception outside the test
+[browser_fail_fp.js]
+fail-if = true
+[browser_fail_pf.js]
+fail-if = true
+[browser_fail_throw.js]
+skip-if = true # fail-if doesnt catch an exception outside the test
+[browser_fail_timeout.js]
+fail-if = true
+[browser_fail_unexpectedTimeout.js]
+fail-if = false
+
--- a/testing/mochitest/tests/browser/moz.build
+++ b/testing/mochitest/tests/browser/moz.build
@ -1,8 +0,0 @@
-# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
-# vim: set filetype=python:
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-BROWSER_CHROME_MANIFESTS += ['browser.ini']
-
--- a/testing/mochitest/tests/moz.build
+++ b/testing/mochitest/tests/moz.build
@ -5,9 +5,8 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.

 DIRS += [
-    'SimpleTest',
-    'browser',
+    'SimpleTest'
 ]

 MOCHITEST_MANIFESTS += ['Harness_sanity/mochitest.ini']
-
+BROWSER_CHROME_MANIFESTS += ['browser/browser.ini']