Bug 758835 - Add ability to mark tests as expected failures in the manifest, r=dburns

testing/marionette/client/marionette/marionette_test.py

@@ -82,10 +82,11 @@ class CommonTestCase(unittest.TestCase):
     match_re = None
     failureException = AssertionError
 
-    def __init__(self, methodName):
+    def __init__(self, methodName, **kwargs):
         unittest.TestCase.__init__(self, methodName)
         self.loglines = []
         self.duration = 0
+        self.expected = kwargs.pop('expected', 'pass')
 
     def _addSkip(self, result, reason):
         addSkip = getattr(result, 'addSkip', None)
@@ -129,7 +130,14 @@ class CommonTestCase(unittest.TestCase):
                 result.addError(self, sys.exc_info())
             else:
                 try:
-                    testMethod()
+                    if self.expected == 'fail':
+                        try:
+                            testMethod()
+                        except Exception:
+                            raise _ExpectedFailure(sys.exc_info())
+                        raise _UnexpectedSuccess
+                    else:
+                        testMethod()
                 except self.failureException:
                     result.addFailure(self, sys.exc_info())
                 except KeyboardInterrupt:
@@ -351,7 +359,7 @@ class MarionetteJSTestCase(CommonTestCase):
         CommonTestCase.__init__(self, methodName)
 
     @classmethod
-    def add_tests_to_suite(cls, mod_name, filepath, suite, testloader, marionette, testvars):
+    def add_tests_to_suite(cls, mod_name, filepath, suite, testloader, marionette, testvars, **kwargs):
         suite.addTest(cls(weakref.ref(marionette), jsFile=filepath))
 
     def runTest(self):

testing/marionette/client/marionette/runtests.py

@@ -524,7 +524,7 @@ class MarionetteTestRunner(object):
             self.marionette.instance = None
         del self.marionette
 
-    def run_test(self, test):
+    def run_test(self, test, expected='pass'):
         if not self.httpd:
             print "starting httpd"
             self.start_httpd()
@@ -584,13 +584,14 @@ class MarionetteTestRunner(object):
             if self.shuffle:
                 random.shuffle(target_tests)
             for i in target_tests:
-                self.run_test(i["path"])
+                self.run_test(i["path"], i["expected"])
                 if self.marionette.check_for_crash():
                     return
             return
 
         self.logger.info('TEST-START %s' % os.path.basename(test))
 
+        self.test_kwargs['expected'] = expected
         for handler in self.test_handlers:
             if handler.match(os.path.basename(test)):
                 handler.add_tests_to_suite(mod_name,

testing/marionette/client/marionette/tests/unit/test_expectedfail.py

@@ -0,0 +1,10 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from marionette_test import MarionetteTestCase
+
+class TestFail(MarionetteTestCase):
+    def test_fails(self):
+        # this test is supposed to fail!
+        self.assertEquals(True, False)

testing/marionette/client/marionette/tests/unit/unit-tests.ini

@@ -11,6 +11,8 @@ b2g = true
 ; true if the test should be skipped
 skip = false
 
+[test_expectedfail.py]
+expected = fail
 [test_getstatus.py]
 [test_import_script.py]
 [test_import_script_reuse_window.py]