Bug 1229992 - add a --stop-on-error cmdline option to the tps test runner. r=whimboo

testing/tps/tps/cli.py

@@ -69,6 +69,10 @@ def main():
                       dest='testfile',
                       default='all_tests.json',
                       help='path to the test file to run [default: %default]')
+    parser.add_option('--stop-on-error',
+                      action='store_true',
+                      dest='stop_on_error',
+                      help='stop running tests after the first failure')
     (options, args) = parser.parse_args()
 
     configfile = options.configfile
@@ -113,6 +117,7 @@ def main():
                         resultfile=options.resultfile,
                         rlock=rlock,
                         testfile=testfile,
+                        stop_on_error=options.stop_on_error,
                       )
     TPS.run_tests()
 

testing/tps/tps/testrunner.py

@@ -118,7 +118,8 @@ class TPSTestRunner(object):
                  mobile=False,
                  rlock=None,
                  resultfile='tps_result.json',
-                 testfile=None):
+                 testfile=None,
+                 stop_on_error=False):
         self.binary = binary
         self.config = config if config else {}
         self.debug = debug
@@ -129,6 +130,7 @@ class TPSTestRunner(object):
         self.rlock = rlock
         self.resultfile = resultfile
         self.testfile = testfile
+        self.stop_on_error = stop_on_error
 
         self.addonversion = None
         self.branch = None
@@ -453,6 +455,9 @@ class TPSTestRunner(object):
                 self.numpassed += 1
             else:
                 self.numfailed += 1
+                if self.stop_on_error:
+                    print '\nTest failed with --stop-on-error specified; not running any more tests.\n'
+                    break
 
         self.mozhttpd.stop()