Backed out 3 changesets (bug 1368674) for Android test failures. a=merge

Backed out changeset ff3c813fcdea (bug 1368674)
Backed out changeset 0d9bb636b9a9 (bug 1368674)
Backed out changeset 1d0202706572 (bug 1368674)

MozReview-Commit-ID: CrCFYIEDH4o

--HG--
extra : source : bf5e8ffae40db7b2c2afcb1851fb8eb80b98e5b6

testing/marionette/client/marionette_driver/marionette.py

@@ -1336,6 +1336,7 @@ class Marionette(object):
 
     @test_name.setter
     def test_name(self, test_name):
+        self._send_message("setTestName", {"value": test_name})
         self._test_name = test_name
 
     def delete_session(self, send_request=True, reset_session_id=False):

testing/marionette/driver.js

@@ -35,6 +35,7 @@ Cu.import("chrome://marionette/content/logging.js");
 Cu.import("chrome://marionette/content/modal.js");
 Cu.import("chrome://marionette/content/proxy.js");
 Cu.import("chrome://marionette/content/session.js");
+Cu.import("chrome://marionette/content/simpletest.js");
 Cu.import("chrome://marionette/content/wait.js");
 
 this.EXPORTED_SYMBOLS = ["GeckoDriver", "Context"];
@@ -892,6 +893,49 @@ GeckoDriver.prototype.execute_ = function (script, args, timeout, opts = {}) {
   }
 };
 
+/**
+ * Execute pure JavaScript.  Used to execute simpletest harness tests,
+ * which are like mochitests only injected using Marionette.
+ *
+ * Scripts are expected to call the {@code finish} global when done.
+ */
+GeckoDriver.prototype.executeJSScript = function* (cmd, resp) {
+  let win = assert.window(this.getCurrentWindow());
+
+  let {script, args, scriptTimeout} = cmd.parameters;
+  scriptTimeout = scriptTimeout || this.timeouts.script;
+
+  let opts = {
+    filename: cmd.parameters.filename,
+    line: cmd.parameters.line,
+    async: cmd.parameters.async,
+  };
+
+  switch (this.context) {
+    case Context.CHROME:
+      let wargs = evaluate.fromJSON(args, this.curBrowser.seenEls, win);
+      let harness = new simpletest.Harness(
+          win,
+          Context.CHROME,
+          this.marionetteLog,
+          scriptTimeout,
+          function() {},
+          this.testName);
+
+      let sb = sandbox.createSimpleTest(win, harness);
+      // TODO(ato): Not sure this is needed:
+      sb = sandbox.augment(sb, new logging.Adapter(this.marionetteLog));
+
+      let res = yield evaluate.sandbox(sb, script, wargs, opts);
+      resp.body.value = evaluate.toJSON(res, this.curBrowser.seenEls);
+      break;
+
+    case Context.CONTENT:
+      resp.body.value = yield this.listener.executeSimpleTest(script, args, scriptTimeout, opts);
+      break;
+  }
+};
+
 /**
  * Navigate to given URL.
  *
@@ -2348,6 +2392,15 @@ GeckoDriver.prototype.sendKeysToElement = function* (cmd, resp) {
   }
 };
 
+/** Sets the test name.  The test name is used for logging purposes. */
+GeckoDriver.prototype.setTestName = function*(cmd, resp) {
+  assert.window(this.getCurrentWindow());
+
+  let val = cmd.parameters.value;
+  this.testName = val;
+  yield this.listener.setTestName({value: val});
+};
+
 /**
  * Clear the text of an element.
  *

testing/marionette/harness/marionette_harness/marionette_test/testcases.py

@@ -3,6 +3,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import imp
+import os
 import re
 import sys
 import time
@@ -19,6 +20,7 @@ from unittest.case import (
 
 from marionette_driver.errors import (
     MarionetteException,
+    ScriptTimeoutException,
     TimeoutException,
 )
 from mozlog import get_default_logger
@@ -58,6 +60,13 @@ class MetaParameterized(type):
         return type.__new__(cls, name, bases, attrs)
 
 
+class JSTest:
+    head_js_re = re.compile(r"MARIONETTE_HEAD_JS(\s*)=(\s*)['|\"](.*?)['|\"];")
+    context_re = re.compile(r"MARIONETTE_CONTEXT(\s*)=(\s*)['|\"](.*?)['|\"];")
+    timeout_re = re.compile(r"MARIONETTE_TIMEOUT(\s*)=(\s*)(\d+);")
+    inactivity_timeout_re = re.compile(r"MARIONETTE_INACTIVITY_TIMEOUT(\s*)=(\s*)(\d+);")
+
+
 class CommonTestCase(unittest.TestCase):
 
     __metaclass__ = MetaParameterized
@@ -226,9 +235,12 @@ class CommonTestCase(unittest.TestCase):
 
     @property
     def test_name(self):
-        return '{0}.py {1}.{2}'.format(self.__class__.__module__,
-                                       self.__class__.__name__,
-                                       self._testMethodName)
+        if hasattr(self, 'jsFile'):
+            return os.path.basename(self.jsFile)
+        else:
+            return '{0}.py {1}.{2}'.format(self.__class__.__module__,
+                                           self.__class__.__name__,
+                                           self._testMethodName)
 
     def id(self):
         # TBPL starring requires that the "test name" field of a failure message
@@ -286,6 +298,127 @@ if (!testUtils.hasOwnProperty("specialPowersObserver")) {
 }
 """)
 
+    def run_js_test(self, filename, marionette=None):
+        """Run a JavaScript test file.
+
+        It collects its set of assertions into the current test's results.
+
+        :param filename: The path to the JavaScript test file to execute.
+                         May be relative to the current script.
+        :param marionette: The Marionette object in which to execute the test.
+                           Defaults to self.marionette.
+        """
+        marionette = marionette or self.marionette
+        if not os.path.isabs(filename):
+            # Find the caller's filename and make the path relative to that.
+            caller_file = sys._getframe(1).f_globals.get('__file__', '')
+            caller_file = os.path.abspath(caller_file)
+            filename = os.path.join(os.path.dirname(caller_file), filename)
+        self.assert_(os.path.exists(filename),
+                     'Script "{}" must exist' .format(filename))
+        original_test_name = self.marionette.test_name
+        self.marionette.test_name = os.path.basename(filename)
+        f = open(filename, 'r')
+        js = f.read()
+        args = []
+
+        head_js = JSTest.head_js_re.search(js)
+        if head_js:
+            head_js = head_js.group(3)
+            head = open(os.path.join(os.path.dirname(filename), head_js), 'r')
+            js = head.read() + js
+
+        context = JSTest.context_re.search(js)
+        if context:
+            context = context.group(3)
+        else:
+            context = 'content'
+
+        if 'SpecialPowers' in js:
+            self.setup_SpecialPowers_observer()
+
+            if context == 'content':
+                js = "var SpecialPowers = window.wrappedJSObject.SpecialPowers;\n" + js
+            else:
+                marionette.execute_script("""
+                if (typeof(SpecialPowers) == 'undefined') {
+                  let loader = Components.classes["@mozilla.org/moz/jssubscript-loader;1"]
+                    .getService(Components.interfaces.mozIJSSubScriptLoader);
+                  loader.loadSubScript("chrome://specialpowers/content/specialpowersAPI.js");
+                  loader.loadSubScript("chrome://specialpowers/content/SpecialPowersObserverAPI.js");
+                  loader.loadSubScript("chrome://specialpowers/content/ChromePowers.js");
+                }
+                """)
+
+        marionette.set_context(context)
+
+        if context != 'chrome':
+            marionette.navigate('data:text/html,<html>test page</html>')
+
+        timeout = JSTest.timeout_re.search(js)
+        if timeout:
+            ms = timeout.group(3)
+            marionette.timeout.script = int(ms) / 1000.0
+
+        inactivity_timeout = JSTest.inactivity_timeout_re.search(js)
+        if inactivity_timeout:
+            inactivity_timeout = int(inactivity_timeout.group(3))
+
+        try:
+            results = marionette.execute_js_script(
+                js,
+                args,
+                inactivity_timeout=inactivity_timeout,
+                filename=os.path.basename(filename)
+            )
+
+            self.assertTrue('timeout' not in filename,
+                            'expected timeout not triggered')
+
+            if 'fail' in filename:
+                self.assertTrue(len(results['failures']) > 0,
+                                "expected test failures didn't occur")
+            else:
+                for failure in results['failures']:
+                    diag = "" if failure.get('diag') is None else failure['diag']
+                    name = ("got false, expected true" if failure.get('name') is None else
+                            failure['name'])
+                    self.logger.test_status(self.test_name, name, 'FAIL',
+                                            message=diag)
+                for failure in results['expectedFailures']:
+                    diag = "" if failure.get('diag') is None else failure['diag']
+                    name = ("got false, expected false" if failure.get('name') is None else
+                            failure['name'])
+                    self.logger.test_status(self.test_name, name, 'FAIL',
+                                            expected='FAIL', message=diag)
+                for failure in results['unexpectedSuccesses']:
+                    diag = "" if failure.get('diag') is None else failure['diag']
+                    name = ("got true, expected false" if failure.get('name') is None else
+                            failure['name'])
+                    self.logger.test_status(self.test_name, name, 'PASS',
+                                            expected='FAIL', message=diag)
+                self.assertEqual(0, len(results['failures']),
+                                 '{} tests failed' .format(len(results['failures'])))
+                if len(results['unexpectedSuccesses']) > 0:
+                    raise _UnexpectedSuccess('')
+                if len(results['expectedFailures']) > 0:
+                    raise _ExpectedFailure((AssertionError, AssertionError(''), None))
+
+            self.assertTrue(results['passed'] +
+                            len(results['failures']) +
+                            len(results['expectedFailures']) +
+                            len(results['unexpectedSuccesses']) > 0,
+                            'no tests run')
+
+        except ScriptTimeoutException:
+            if 'timeout' in filename:
+                # expected exception
+                pass
+            else:
+                self.loglines = marionette.get_logs()
+                raise
+        self.marionette.test_name = original_test_name
+
 
 class MarionetteTestCase(CommonTestCase):
 
@@ -334,8 +467,9 @@ class MarionetteTestCase(CommonTestCase):
     def setUp(self):
         super(MarionetteTestCase, self).setUp()
         self.marionette.test_name = self.test_name
-        self.marionette.execute_script("log('TEST-START: {0}')"
-                                       .format(self.test_name),
+        self.marionette.execute_script("log('TEST-START: {0}:{1}')"
+                                       .format(self.filepath.replace('\\', '\\\\'),
+                                               self.methodName),
                                        sandbox="simpletest")
 
     def tearDown(self):
@@ -347,8 +481,9 @@ class MarionetteTestCase(CommonTestCase):
         if not self.marionette.crashed:
             try:
                 self.marionette.clear_imported_scripts()
-                self.marionette.execute_script("log('TEST-END: {0}')"
-                                               .format(self.test_name),
+                self.marionette.execute_script("log('TEST-END: {0}:{1}')"
+                                               .format(self.filepath.replace('\\', '\\\\'),
+                                                       self.methodName),
                                                sandbox="simpletest")
                 self.marionette.test_name = None
             except (MarionetteException, IOError):

testing/marionette/harness/marionette_harness/runner/base.py

@@ -183,6 +183,8 @@ class MarionetteTestResult(StructuredTestResult, TestResultCollection):
             return '\n'.join((str(test), doc_first_line))
         else:
             desc = str(test)
+            if hasattr(test, 'jsFile'):
+                desc = "{0}, {1}".format(test.jsFile, desc)
             return desc
 
     def printLogs(self, test):
@@ -604,7 +606,7 @@ class BaseMarionetteTestRunner(object):
     def filename_pattern(self):
         if self._filename_pattern is None:
             self._filename_pattern = re.compile(
-                "^test(((_.+?)+?\.((py))))$")
+                "^test(((_.+?)+?\.((py)|(js)))|(([A-Z].*?)+?\.js))$")
 
         return self._filename_pattern
 
@@ -801,7 +803,7 @@ class BaseMarionetteTestRunner(object):
                          if not self._is_filename_valid(t['filepath'])]
         if invalid_tests:
             raise Exception("Test file names must be of the form "
-                            "'test_something.py'."
+                            "'test_something.py', 'test_something.js', or 'testSomething.js'."
                             " Invalid test names:\n  {}".format('\n  '.join(invalid_tests)))
 
     def _is_filename_valid(self, filename):

testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py

@@ -282,8 +282,8 @@ def test_add_test_module(runner):
 def test_add_test_directory(runner):
     test_dir = 'path/to/tests'
     dir_contents = [
-        (test_dir, ('subdir',), ('test_a.py', 'bad_test_a.py')),
-        (test_dir + '/subdir', (), ('test_b.py', 'bad_test_b.py')),
+        (test_dir, ('subdir',), ('test_a.py', 'test_a.js', 'bad_test_a.py', 'bad_test_a.js')),
+        (test_dir + '/subdir', (), ('test_b.py', 'test_b.js', 'bad_test_b.py', 'bad_test_b.js')),
     ]
     tests = list(dir_contents[0][2] + dir_contents[1][2])
     assert len(runner.tests) == 0
@@ -294,7 +294,7 @@ def test_add_test_directory(runner):
     assert isdir.called and walk.called
     for test in runner.tests:
         assert test_dir in test['filepath']
-    assert len(runner.tests) == 2
+    assert len(runner.tests) == 4
 
 
 @pytest.mark.parametrize("test_files_exist", [True, False])
@@ -404,10 +404,10 @@ def test_add_tests(mock_runner):
 
 
 def test_catch_invalid_test_names(runner):
-    good_tests = [u'test_ok.py', u'test_is_ok.py']
-    bad_tests = [u'bad_test.py', u'testbad.py', u'_test_bad.py',
-                 u'test_bad.notpy', u'test_bad',
-                 u'test.py', u'test_.py']
+    good_tests = [u'test_ok.py', u'test_is_ok.py', u'test_is_ok.js', u'testIsOk.js']
+    bad_tests = [u'bad_test.py', u'testbad.py', u'_test_bad.py', u'testBad.notjs',
+                 u'test_bad.notpy', u'test_bad', u'testbad.js', u'badtest.js',
+                 u'test.py', u'test_.py', u'test.js', u'test_.js']
     with pytest.raises(Exception) as exc:
         runner._add_tests(good_tests + bad_tests)
     msg = exc.value.message

testing/marionette/harness/marionette_harness/tests/unit/test_chrome_async_finish.js

@@ -0,0 +1,6 @@
+MARIONETTE_TIMEOUT = 60000;
+MARIONETTE_CONTEXT = "chrome";
+ok(true);
+(function () {
+  finish();
+})();

testing/marionette/harness/marionette_harness/tests/unit/test_execute_script.py

@@ -29,6 +29,19 @@ globals = set([
               ])
 
 
+class TestExecuteSimpleTestContent(MarionetteTestCase):
+    def test_stack_trace(self):
+        try:
+            self.marionette.execute_js_script("""
+                let a = 1;
+                throwHere();
+                """, filename="file.js")
+            self.assertFalse(True)
+        except errors.JavascriptException as e:
+            self.assertIn("throwHere is not defined", e.message)
+            self.assertIn("@file.js:2", e.stacktrace)
+
+
 class TestExecuteContent(MarionetteTestCase):
 
     def assert_is_defined(self, property, sandbox="default"):

testing/marionette/harness/marionette_harness/tests/unit/test_simpletest_chrome.js

@@ -0,0 +1,12 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+MARIONETTE_TIMEOUT = 1000;
+MARIONETTE_CONTEXT = 'chrome';
+
+is(2, 2, "test for is()");
+isnot(2, 3, "test for isnot()");
+ok(2 == 2, "test for ok()");
+finish();
+

testing/marionette/harness/marionette_harness/tests/unit/test_simpletest_fail.js

@@ -0,0 +1,16 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+MARIONETTE_TIMEOUT = 1000;
+
+/* this test will fail */
+
+setTimeout(function() { 
+    is(1, 2, "is(1,2) should fail", TEST_UNEXPECTED_FAIL, TEST_PASS); 
+    finish();
+}, 100);
+isnot(1, 1, "isnot(1,1) should fail", TEST_UNEXPECTED_FAIL, TEST_PASS);
+ok(1 == 2, "ok(1==2) should fail", TEST_UNEXPECTED_FAIL, TEST_PASS);
+
+

testing/marionette/harness/marionette_harness/tests/unit/test_simpletest_pass.js

@@ -0,0 +1,12 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+MARIONETTE_TIMEOUT = 1000;
+
+is(2, 2, "test for is()");
+isnot(2, 3, "test for isnot()");
+ok(2 == 2, "test for ok()");
+
+setTimeout(finish, 100);
+

testing/marionette/harness/marionette_harness/tests/unit/test_simpletest_sanity.py

@@ -0,0 +1,107 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from marionette_harness import MarionetteTestCase
+
+
+class SimpletestSanityTest(MarionetteTestCase):
+    callFinish = "return finish();"
+
+    def run_sync(self, test):
+        return self.marionette.execute_js_script(test, async=False)
+
+    def run_async(self, test):
+        return self.marionette.execute_js_script(test)
+
+    def test_is(self):
+        def runtests():
+            sentFail1 = "is(true, false, 'isTest1', TEST_UNEXPECTED_FAIL, TEST_PASS);" + self.callFinish
+            sentFail2 = "is(true, false, 'isTest2', TEST_UNEXPECTED_FAIL, TEST_PASS);" + self.callFinish
+            sentPass1 = "is(true, true, 'isTest3');" + self.callFinish
+            sentPass2 = "is(true, true, 'isTest4');" + self.callFinish
+
+            self.assertEqual(1, len(self.run_sync(sentFail1)["failures"]))
+            self.assertEqual(0, self.run_sync(sentFail2)["passed"])
+            self.assertEqual(1, self.run_sync(sentPass1)["passed"])
+            self.assertEqual(0, len(self.run_sync(sentPass2)["failures"]))
+
+            self.marionette.timeout.script = 1
+            self.assertEqual(1, len(self.run_async(sentFail1)["failures"]))
+            self.assertEqual(0, self.run_async(sentFail2)["passed"])
+            self.assertEqual(1, self.run_async(sentPass1)["passed"])
+            self.assertEqual(0, len(self.run_async(sentPass2)["failures"]))
+
+        self.marionette.set_context("content")
+        runtests()
+        self.marionette.set_context("chrome")
+        runtests()
+
+    def test_isnot(self):
+        def runtests():
+           sentFail1 = "isnot(true, true, 'isnotTest3', TEST_UNEXPECTED_FAIL, TEST_PASS);" + self.callFinish
+           sentFail2 = "isnot(true, true, 'isnotTest4', TEST_UNEXPECTED_FAIL, TEST_PASS);" + self.callFinish
+           sentPass1 = "isnot(true, false, 'isnotTest1');" + self.callFinish
+           sentPass2 = "isnot(true, false, 'isnotTest2');" + self.callFinish
+
+           self.assertEqual(1, len(self.run_sync(sentFail1)["failures"]));
+           self.assertEqual(0, self.run_sync(sentFail2)["passed"]);
+           self.assertEqual(0, len(self.run_sync(sentPass1)["failures"]));
+           self.assertEqual(1, self.run_sync(sentPass2)["passed"]);
+
+           self.marionette.timeout.script = 1
+           self.assertEqual(1, len(self.run_async(sentFail1)["failures"]));
+           self.assertEqual(0, self.run_async(sentFail2)["passed"]);
+           self.assertEqual(0, len(self.run_async(sentPass1)["failures"]));
+           self.assertEqual(1, self.run_async(sentPass2)["passed"]);
+
+        self.marionette.set_context("content")
+        runtests()
+        self.marionette.set_context("chrome")
+        runtests()
+
+    def test_ok(self):
+        def runtests():
+            sentFail1 = "ok(1==2, 'testOk1', TEST_UNEXPECTED_FAIL, TEST_PASS);" + self.callFinish
+            sentFail2 = "ok(1==2, 'testOk2', TEST_UNEXPECTED_FAIL, TEST_PASS);" + self.callFinish
+            sentPass1 = "ok(1==1, 'testOk3');" + self.callFinish
+            sentPass2 = "ok(1==1, 'testOk4');" + self.callFinish
+
+            self.assertEqual(1, len(self.run_sync(sentFail1)["failures"]));
+            self.assertEqual(0, self.run_sync(sentFail2)["passed"]);
+            self.assertEqual(0, len(self.run_sync(sentPass1)["failures"]));
+            self.assertEqual(1, self.run_sync(sentPass2)["passed"]);
+
+            self.marionette.timeout.script = 1
+            self.assertEqual(1, len(self.run_async(sentFail1)["failures"]));
+            self.assertEqual(0, self.run_async(sentFail2)["passed"]);
+            self.assertEqual(0, len(self.run_async(sentPass1)["failures"]));
+            self.assertEqual(1, self.run_async(sentPass2)["passed"]);
+
+        self.marionette.set_context("content")
+        runtests()
+        self.marionette.set_context("chrome")
+        runtests()
+
+    def test_todo(self):
+        def runtests():
+            sentFail1 = "todo(1==1, 'testTodo1', TEST_UNEXPECTED_PASS, TEST_KNOWN_FAIL);" + self.callFinish
+            sentFail2 = "todo(1==1, 'testTodo2', TEST_UNEXPECTED_PASS, TEST_KNOWN_FAIL);" + self.callFinish
+            sentPass1 = "todo(1==2, 'testTodo3');" + self.callFinish
+            sentPass2 = "todo(1==2, 'testTodo4');" + self.callFinish
+
+            self.assertEqual(1, len(self.run_sync(sentFail1)["unexpectedSuccesses"]));
+            self.assertEqual(0, len(self.run_sync(sentFail2)["expectedFailures"]));
+            self.assertEqual(0, len(self.run_sync(sentPass1)["unexpectedSuccesses"]));
+            self.assertEqual(1, len(self.run_sync(sentPass2)["expectedFailures"]));
+
+            self.marionette.timeout.script = 1
+            self.assertEqual(1, len(self.run_async(sentFail1)["unexpectedSuccesses"]));
+            self.assertEqual(0, len(self.run_async(sentFail2)["expectedFailures"]));
+            self.assertEqual(0, len(self.run_async(sentPass1)["unexpectedSuccesses"]));
+            self.assertEqual(1, len(self.run_async(sentPass2)["expectedFailures"]));
+
+        self.marionette.set_context("content")
+        runtests()
+        self.marionette.set_context("chrome")
+        runtests()

testing/marionette/harness/marionette_harness/tests/unit/test_simpletest_timeout.js

@@ -0,0 +1,16 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+MARIONETTE_TIMEOUT = 100;
+
+/* this test will timeout */
+
+function do_test() {
+  is(1, 1);
+  isnot(1, 2);
+  ok(1 == 1);
+  finish();
+}
+
+setTimeout(do_test, 1000);

testing/marionette/harness/marionette_harness/tests/unit/unit-tests.ini

@@ -33,6 +33,7 @@ skip-if = true # "Bug 896046"
 
 [test_execute_async_script.py]
 [test_execute_script.py]
+[test_simpletest_fail.js]
 [test_element_retrieval.py]
 [test_findelement_chrome.py]
 skip-if = appname == 'fennec'
@@ -45,6 +46,10 @@ skip-if = appname == 'fennec'
 [test_single_finger_desktop.py]
 skip-if = appname == 'fennec' || os == "win" # Bug 1025040
 
+[test_simpletest_pass.js]
+[test_simpletest_sanity.py]
+[test_simpletest_chrome.js]
+[test_simpletest_timeout.js]
 [test_anonymous_content.py]
 skip-if = appname == 'fennec'
 [test_switch_frame.py]
@@ -88,6 +93,7 @@ skip-if = appname == 'fennec'
 [test_date_time_value.py]
 [test_getactiveframe_oop.py]
 skip-if = true # Bug 925688
+[test_chrome_async_finish.js]
 [test_screen_orientation.py]
 [test_errors.py]
 

testing/marionette/jar.mn

@@ -13,6 +13,7 @@ marionette.jar:
   content/accessibility.js (accessibility.js)
   content/listener.js (listener.js)
   content/element.js (element.js)
+  content/simpletest.js (simpletest.js)
   content/frame.js (frame.js)
   content/cert.js (cert.js)
   content/event.js  (event.js)

testing/marionette/listener.js

@@ -27,6 +27,7 @@ Cu.import("chrome://marionette/content/logging.js");
 Cu.import("chrome://marionette/content/navigate.js");
 Cu.import("chrome://marionette/content/proxy.js");
 Cu.import("chrome://marionette/content/session.js");
+Cu.import("chrome://marionette/content/simpletest.js");
 
 Cu.import("resource://gre/modules/FileUtils.jsm");
 Cu.import("resource://gre/modules/Preferences.jsm");
@@ -489,6 +490,7 @@ var deleteCookieFn = dispatch(deleteCookie);
 var deleteAllCookiesFn = dispatch(deleteAllCookies);
 var executeFn = dispatch(execute);
 var executeInSandboxFn = dispatch(executeInSandbox);
+var executeSimpleTestFn = dispatch(executeSimpleTest);
 var sendKeysToElementFn = dispatch(sendKeysToElement);
 
 /**
@@ -498,6 +500,7 @@ function startListeners() {
   addMessageListenerId("Marionette:newSession", newSession);
   addMessageListenerId("Marionette:execute", executeFn);
   addMessageListenerId("Marionette:executeInSandbox", executeInSandboxFn);
+  addMessageListenerId("Marionette:executeSimpleTest", executeSimpleTestFn);
   addMessageListenerId("Marionette:singleTap", singleTapFn);
   addMessageListenerId("Marionette:performActions", performActionsFn);
   addMessageListenerId("Marionette:releaseActions", releaseActionsFn);
@@ -532,6 +535,7 @@ function startListeners() {
   addMessageListenerId("Marionette:deleteSession", deleteSession);
   addMessageListenerId("Marionette:sleepSession", sleepSession);
   addMessageListenerId("Marionette:getAppCacheStatus", getAppCacheStatus);
+  addMessageListenerId("Marionette:setTestName", setTestName);
   addMessageListenerId("Marionette:takeScreenshot", takeScreenshotFn);
   addMessageListenerId("Marionette:addCookie", addCookieFn);
   addMessageListenerId("Marionette:getCookies", getCookiesFn);
@@ -572,6 +576,7 @@ function deleteSession(msg) {
   removeMessageListenerId("Marionette:newSession", newSession);
   removeMessageListenerId("Marionette:execute", executeFn);
   removeMessageListenerId("Marionette:executeInSandbox", executeInSandboxFn);
+  removeMessageListenerId("Marionette:executeSimpleTest", executeSimpleTestFn);
   removeMessageListenerId("Marionette:singleTap", singleTapFn);
   removeMessageListenerId("Marionette:performActions", performActionsFn);
   removeMessageListenerId("Marionette:releaseActions", releaseActionsFn);
@@ -606,6 +611,7 @@ function deleteSession(msg) {
   removeMessageListenerId("Marionette:deleteSession", deleteSession);
   removeMessageListenerId("Marionette:sleepSession", sleepSession);
   removeMessageListenerId("Marionette:getAppCacheStatus", getAppCacheStatus);
+  removeMessageListenerId("Marionette:setTestName", setTestName);
   removeMessageListenerId("Marionette:takeScreenshot", takeScreenshotFn);
   removeMessageListenerId("Marionette:addCookie", addCookieFn);
   removeMessageListenerId("Marionette:getCookies", getCookiesFn);
@@ -757,6 +763,39 @@ function* executeInSandbox(script, args, timeout, opts) {
   return evaluate.toJSON(res, seenEls);
 }
 
+function* executeSimpleTest(script, args, timeout, opts) {
+  opts.timeout = timeout;
+  let win = curContainer.frame;
+
+  let harness = new simpletest.Harness(
+      win,
+      "content",
+      contentLog,
+      timeout,
+      marionetteTestName);
+  let sb = sandbox.createSimpleTest(curContainer.frame, harness);
+  // TODO(ato): Not sure this is needed:
+  sb = sandbox.augment(sb, new logging.Adapter(contentLog));
+
+  let wargs = evaluate.fromJSON(
+      args, seenEls, curContainer.frame, curContainer.shadowRoot);
+  let evaluatePromise = evaluate.sandbox(sb, script, wargs, opts);
+
+  let res = yield evaluatePromise;
+  sendSyncMessage(
+      "Marionette:shareData",
+      {log: evaluate.toJSON(contentLog.get(), seenEls)});
+  return evaluate.toJSON(res, seenEls);
+}
+
+/**
+ * Sets the test name, used in logging messages.
+ */
+function setTestName(msg) {
+  marionetteTestName = msg.json.value;
+  sendOk(msg.json.command_id);
+}
+
 /**
  * This function creates a touch event given a touch type and a touch
  */

testing/marionette/simpletest.js

@@ -0,0 +1,208 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const {utils: Cu} = Components;
+
+Cu.import("chrome://marionette/content/error.js");
+
+this.EXPORTED_SYMBOLS = ["simpletest"];
+
+this.simpletest = {};
+
+/**
+ * The simpletest harness, exposed in the script evaluation sandbox.
+ */
+simpletest.Harness = class {
+  constructor(window, context, contentLogger, timeout, testName) {
+    this.window = window;
+    this.tests = [];
+    this.logger = contentLogger;
+    this.context = context;
+    this.timeout = timeout;
+    this.testName = testName;
+    this.TEST_UNEXPECTED_FAIL = "TEST-UNEXPECTED-FAIL";
+    this.TEST_UNEXPECTED_PASS = "TEST-UNEXPECTED-PASS";
+    this.TEST_PASS = "TEST-PASS";
+    this.TEST_KNOWN_FAIL = "TEST-KNOWN-FAIL";
+  }
+
+  get exports() {
+    return new Map([
+      ["ok", this.ok.bind(this)],
+      ["is", this.is.bind(this)],
+      ["isnot", this.isnot.bind(this)],
+      ["todo", this.todo.bind(this)],
+      ["log", this.log.bind(this)],
+      ["getLogs", this.getLogs.bind(this)],
+      ["generate_results", this.generate_results.bind(this)],
+      ["waitFor", this.waitFor.bind(this)],
+      ["TEST_PASS", this.TEST_PASS],
+      ["TEST_KNOWN_FAIL", this.TEST_KNOWN_FAIL],
+      ["TEST_UNEXPECTED_FAIL", this.TEST_UNEXPECTED_FAIL],
+      ["TEST_UNEXPECTED_PASS", this.TEST_UNEXPECTED_PASS],
+    ]);
+  }
+
+  addTest(condition, name, passString, failString, diag, state) {
+    let test = {
+      result: !!condition,
+      name: name,
+      diag: diag,
+      state: state
+    };
+    this.logResult(
+        test,
+        typeof passString == "undefined" ? this.TEST_PASS : passString,
+        typeof failString == "undefined" ? this.TEST_UNEXPECTED_FAIL : failString);
+    this.tests.push(test);
+  }
+
+  ok(condition, name, passString, failString) {
+    let diag = `${this.repr(condition)} was ${!!condition}, expected true`;
+    this.addTest(condition, name, passString, failString, diag);
+  }
+
+  is(a, b, name, passString, failString) {
+    let pass = (a == b);
+    let diag = pass ? this.repr(a) + " should equal " + this.repr(b)
+                    : "got " + this.repr(a) + ", expected " + this.repr(b);
+    this.addTest(pass, name, passString, failString, diag);
+  }
+
+  isnot(a, b, name, passString, failString) {
+    let pass = (a != b);
+    let diag = pass ? this.repr(a) + " should not equal " + this.repr(b)
+                    : "didn't expect " + this.repr(a) + ", but got it";
+    this.addTest(pass, name, passString, failString, diag);
+  }
+
+  todo(condition, name, passString, failString) {
+    let diag = this.repr(condition) + " was expected false";
+    this.addTest(!condition,
+                 name,
+                 typeof(passString) == "undefined" ? this.TEST_KNOWN_FAIL : passString,
+                 typeof(failString) == "undefined" ? this.TEST_UNEXPECTED_FAIL : failString,
+                 diag,
+                 "todo");
+  }
+
+  log(msg, level) {
+    dump("MARIONETTE LOG: " + (level ? level : "INFO") + ": " + msg + "\n");
+    if (this.logger) {
+      this.logger.log(msg, level);
+    }
+  }
+
+  // TODO(ato): Suspect this isn't used anywhere
+  getLogs() {
+    if (this.logger) {
+      return this.logger.get();
+    }
+  }
+
+  generate_results() {
+    let passed = 0;
+    let failures = [];
+    let expectedFailures = [];
+    let unexpectedSuccesses = [];
+    for (let i in this.tests) {
+      let isTodo = (this.tests[i].state == "todo");
+      if(this.tests[i].result) {
+        if (isTodo) {
+          expectedFailures.push({'name': this.tests[i].name, 'diag': this.tests[i].diag});
+        }
+        else {
+          passed++;
+        }
+      }
+      else {
+        if (isTodo) {
+          unexpectedSuccesses.push({'name': this.tests[i].name, 'diag': this.tests[i].diag});
+        }
+        else {
+          failures.push({'name': this.tests[i].name, 'diag': this.tests[i].diag});
+        }
+      }
+    }
+    // Reset state in case this object is reused for more tests.
+    this.tests = [];
+    return {
+      passed: passed,
+      failures: failures,
+      expectedFailures: expectedFailures,
+      unexpectedSuccesses: unexpectedSuccesses,
+    };
+  }
+
+  logToFile(file) {
+    //TODO
+  }
+
+  logResult(test, passString, failString) {
+    //TODO: dump to file
+    let resultString = test.result ? passString : failString;
+    let diagnostic = test.name + (test.diag ? " - " + test.diag : "");
+    let msg = resultString + " | " + this.testName + " | " + diagnostic;
+    dump("MARIONETTE TEST RESULT:" + msg + "\n");
+  }
+
+  repr(o) {
+    if (typeof o == "undefined") {
+      return "undefined";
+    } else if (o === null) {
+      return "null";
+    }
+
+    try {
+        if (typeof o.__repr__ == "function") {
+          return o.__repr__();
+        } else if (typeof o.repr == "function" && o.repr !== arguments.callee) {
+          return o.repr();
+        }
+   } catch (e) {}
+
+   try {
+      if (typeof o.NAME === "string" &&
+          (o.toString === Function.prototype.toString || o.toString === Object.prototype.toString)) {
+        return o.NAME;
+      }
+    } catch (e) {}
+
+    let ostring;
+    try {
+      ostring = (o + "");
+    } catch (e) {
+      return "[" + typeof(o) + "]";
+    }
+
+    if (typeof o == "function") {
+      o = ostring.replace(/^\s+/, "");
+      let idx = o.indexOf("{");
+      if (idx != -1) {
+        o = o.substr(0, idx) + "{...}";
+      }
+    }
+    return ostring;
+  }
+
+  waitFor(callback, test, timeout) {
+    if (test()) {
+      callback();
+      return;
+    }
+
+    let now = new Date();
+    let deadline = (timeout instanceof Date) ? timeout :
+        new Date(now.valueOf() + (typeof timeout == "undefined" ? this.timeout : timeout));
+    if (deadline <= now) {
+      dump("waitFor timeout: " + test.toString() + "\n");
+      // the script will timeout here, so no need to raise a separate
+      // timeout exception
+      return;
+    }
+    this.window.setTimeout(this.waitFor.bind(this), 100, callback, test, deadline);
+  }
+};