diff --git a/testing/mochitest/browser-test.js b/testing/mochitest/browser-test.js index 941ac0d0380d..de96972692ed 100644 --- a/testing/mochitest/browser-test.js +++ b/testing/mochitest/browser-test.js @@ -1259,7 +1259,10 @@ function testResult({ name, pass, todo, ex, stack, allowFailure }) { if (allowFailure && !pass) { this.allowedFailure = true; this.pass = true; - this.todo = true; + this.todo = false; + } else if (allowFailure && pass) { + this.pass = true; + this.todo = false; } else { this.pass = !!pass; this.todo = todo; diff --git a/testing/mochitest/runtests.py b/testing/mochitest/runtests.py index 32da88e80fb3..43f87506954d 100644 --- a/testing/mochitest/runtests.py +++ b/testing/mochitest/runtests.py @@ -2549,16 +2549,26 @@ toolbar#nav-bar { # record post-test information if status: self.message_logger.dump_buffered() - if crashAsPass: - self.log.info( - "TEST-PASS | %s | application terminated with exit code %s" - % (self.lastTestSeen, status) - ) + msg = ("application terminated with exit code %s" % status,) + # self.message_logger.is_test_running indicates we need to send a test_end + if crashAsPass and self.message_logger.is_test_running: + # this works for browser-chrome, mochitest-plain has status=0 + message = { + "action": "test_end", + "status": "CRASH", + "expected": "CRASH", + "thread": None, + "pid": None, + "source": "mochitest", + "time": int(time.time()) * 1000, + "test": self.lastTestSeen, + "message": msg, + } + # need to send a test_end in order to have mozharness process messages properly + # this requires a custom message vs log.error/log.warning/etc. + self.message_logger.process_message(message) else: - self.log.error( - "TEST-UNEXPECTED-FAIL | %s | application terminated with exit code %s" - % (self.lastTestSeen, status) - ) + self.log.error(msg) else: self.lastTestSeen = "Main app process exited normally" @@ -2576,6 +2586,7 @@ toolbar#nav-bar { quiet = False if crashAsPass: quiet = True + minidump_path = os.path.join(self.profile.profile, "minidumps") crash_count = mozcrash.log_crashes( self.log, @@ -2585,12 +2596,27 @@ toolbar#nav-bar { quiet=quiet, ) - if crash_count or zombieProcesses: - status = 1 - if crashAsPass: + # self.message_logger.is_test_running indicates we need a test_end message + if crash_count > 0 and self.message_logger.is_test_running: + # this works for browser-chrome, mochitest-plain has status=0 + message = { + "action": "test_end", + "status": "CRASH", + "expected": "CRASH", + "thread": None, + "pid": None, + "source": "mochitest", + "time": int(time.time()) * 1000, + "test": self.lastTestSeen, + "message": "application terminated with exit code 0", + } + # need to send a test_end in order to have mozharness process messages properly + # this requires a custom message vs log.error/log.warning/etc. + self.message_logger.process_message(message) status = 0 - + elif crash_count or zombieProcesses: + status = 1 finally: # cleanup if os.path.exists(processLog): @@ -2934,6 +2960,14 @@ toolbar#nav-bar { e10s_mode = "e10s" if options.e10s else "non-e10s" + # for failure mode: where browser window has crashed and we have no reported results + if ( + self.countpass == self.countfail == self.counttodo == 0 + and options.crashAsPass + ): + self.countpass = 1 + self.result = 0 + # printing total number of tests if options.flavor == "browser": print("TEST-INFO | checking window state") @@ -3101,6 +3135,7 @@ toolbar#nav-bar { mozinfo.info["debug"] and options.flavor == "browser" and options.subsuite != "thunderbird" + and not options.crashAsPass ) self.start_script_kwargs["flavor"] = self.normflavor(options.flavor) @@ -3201,6 +3236,10 @@ toolbar#nav-bar { ignoreMissingLeaks = options.ignoreMissingLeaks leakThresholds = options.leakThresholds + if options.crashAsPass: + ignoreMissingLeaks.append("tab") + ignoreMissingLeaks.append("socket") + # Stop leak detection if m-bc code coverage is enabled # by maxing out the leak threshold for all processes. if options.jscov_dir_prefix: diff --git a/testing/mochitest/tests/SimpleTest/SimpleTest.js b/testing/mochitest/tests/SimpleTest/SimpleTest.js index ea765a6bdc75..4b86612a77ed 100644 --- a/testing/mochitest/tests/SimpleTest/SimpleTest.js +++ b/testing/mochitest/tests/SimpleTest/SimpleTest.js @@ -425,7 +425,7 @@ SimpleTest.record = function(condition, name, diag, stack, expected) { if (SimpleTest.expected == "fail") { if (!test.result) { SimpleTest.num_failed++; - test.todo = true; + test.result = true; } successInfo = { status: "PASS", diff --git a/testing/mochitest/tests/python/conftest.py b/testing/mochitest/tests/python/conftest.py index 6cf75e937835..38e927fb699a 100644 --- a/testing/mochitest/tests/python/conftest.py +++ b/testing/mochitest/tests/python/conftest.py @@ -38,7 +38,12 @@ def runtests(setup_test_harness, binary, parser, request): if "flavor" in request.fixturenames: flavor = request.getfixturevalue("flavor") + runFailures = "" + if "runFailures" in request.fixturenames: + runFailures = request.getfixturevalue("runFailures") + setup_test_harness(*setup_args, flavor=flavor) + runtests = pytest.importorskip("runtests") mochitest_root = runtests.SCRIPT_DIR @@ -58,11 +63,16 @@ def runtests(setup_test_harness, binary, parser, request): { "app": binary, "flavor": flavor, + "runFailures": runFailures, "keep_open": False, "log_raw": [buf], } ) + if runFailures == "selftest": + options["crashAsPass"] = True + options["timeoutAsPass"] = True + if not os.path.isdir(runtests.build_obj.bindir): package_root = os.path.dirname(mochitest_root) options.update( @@ -85,6 +95,7 @@ def runtests(setup_test_harness, binary, parser, request): # add a dummy manifest file because mochitest expects it "manifest": os.path.join(test_root, manifest_name), "manifest_relpath": manifest_name, + "skip-if": runFailures, } def inner(*tests, **opts): diff --git a/testing/mochitest/tests/python/test_mochitest_integration.py b/testing/mochitest/tests/python/test_mochitest_integration.py index ff6ee9799553..ae560c9d6485 100644 --- a/testing/mochitest/tests/python/test_mochitest_integration.py +++ b/testing/mochitest/tests/python/test_mochitest_integration.py @@ -32,108 +32,172 @@ def test_name(request): return inner +@pytest.mark.parametrize("runFailures", ["selftest", ""]) @pytest.mark.parametrize("flavor", ["plain", "browser-chrome"]) -def test_output_pass(flavor, runtests, test_name): - status, lines = runtests(test_name("pass")) - assert status == 0 +def test_output_pass(flavor, runFailures, runtests, test_name): + extra_opts = {} + results = { + "status": 1 if runFailures else 0, + "tbpl_status": TBPL_WARNING if runFailures else TBPL_SUCCESS, + "log_level": (INFO, WARNING), + "lines": 2 if runFailures else 1, + "line_status": "PASS", + } + if runFailures: + extra_opts["runFailures"] = runFailures + extra_opts["crashAsPass"] = True + extra_opts["timeoutAsPass"] = True + + status, lines = runtests(test_name("pass"), **extra_opts) + assert status == results["status"] tbpl_status, log_level, summary = get_mozharness_status(lines, status) - assert tbpl_status == TBPL_SUCCESS - assert log_level in (INFO, WARNING) + assert tbpl_status == results["tbpl_status"] + assert log_level in results["log_level"] lines = filter_action("test_status", lines) - assert len(lines) == 1 - assert lines[0]["status"] == "PASS" + assert len(lines) == results["lines"] + assert lines[0]["status"] == results["line_status"] +@pytest.mark.parametrize("runFailures", ["selftest", ""]) @pytest.mark.parametrize("flavor", ["plain", "browser-chrome"]) -def test_output_fail(flavor, runtests, test_name): - status, lines = runtests(test_name("fail")) - assert status == 1 +def test_output_fail(flavor, runFailures, runtests, test_name): + extra_opts = {} + results = { + "status": 0 if runFailures else 1, + "tbpl_status": TBPL_SUCCESS if runFailures else TBPL_WARNING, + "log_level": (INFO, WARNING), + "lines": 1, + "line_status": "PASS" if runFailures else "FAIL", + } + if runFailures: + extra_opts["runFailures"] = runFailures + extra_opts["crashAsPass"] = True + extra_opts["timeoutAsPass"] = True + + status, lines = runtests(test_name("fail"), **extra_opts) + assert status == results["status"] tbpl_status, log_level, summary = get_mozharness_status(lines, status) - assert tbpl_status == TBPL_WARNING - assert log_level == WARNING + assert tbpl_status == results["tbpl_status"] + assert log_level in results["log_level"] lines = filter_action("test_status", lines) - - assert len(lines) == 1 - assert lines[0]["status"] == "FAIL" + assert len(lines) == results["lines"] + assert lines[0]["status"] == results["line_status"] @pytest.mark.skip_mozinfo("!crashreporter") +@pytest.mark.parametrize("runFailures", ["selftest", ""]) @pytest.mark.parametrize("flavor", ["plain", "browser-chrome"]) -def test_output_crash(flavor, runtests, test_name): +def test_output_crash(flavor, runFailures, runtests, test_name): + extra_opts = {} + results = { + "status": 0 if runFailures else 1, + "tbpl_status": TBPL_FAILURE, + "log_level": ERROR, + "lines": 1 if runFailures else 0, + } + if runFailures: + extra_opts["runFailures"] = runFailures + extra_opts["crashAsPass"] = True + extra_opts["timeoutAsPass"] = True + # bug 1443327 - we do not set MOZ_CRASHREPORTER_SHUTDOWN for browser-chrome + # the error regex's don't pick this up as a failure + if flavor == "browser-chrome": + results["tbpl_status"] = TBPL_SUCCESS + results["log_level"] = (INFO, WARNING) + status, lines = runtests( - test_name("crash"), environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"] + test_name("crash"), environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"], **extra_opts ) - assert status == 1 + assert status == results["status"] tbpl_status, log_level, summary = get_mozharness_status(lines, status) - assert tbpl_status == TBPL_FAILURE - assert log_level == ERROR + assert tbpl_status == results["tbpl_status"] + assert log_level in results["log_level"] - crash = filter_action("crash", lines) - assert len(crash) == 1 - assert crash[0]["action"] == "crash" - assert crash[0]["signature"] - assert crash[0]["minidump_path"] + if not runFailures: + crash = filter_action("crash", lines) + assert len(crash) == 1 + assert crash[0]["action"] == "crash" + assert crash[0]["signature"] + assert crash[0]["minidump_path"] lines = filter_action("test_end", lines) - assert len(lines) == 0 + assert len(lines) == results["lines"] @pytest.mark.skip_mozinfo("!asan") +@pytest.mark.parametrize("runFailures", [""]) @pytest.mark.parametrize("flavor", ["plain"]) -def test_output_asan(flavor, runtests, test_name): +def test_output_asan(flavor, runFailures, runtests, test_name): + extra_opts = {} + results = {"status": 1, "tbpl_status": TBPL_FAILURE, "log_level": ERROR, "lines": 0} + status, lines = runtests( - test_name("crash"), environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"] + test_name("crash"), environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"], **extra_opts ) - assert status == 1 + assert status == results["status"] tbpl_status, log_level, summary = get_mozharness_status(lines, status) - assert tbpl_status == TBPL_FAILURE - assert log_level == ERROR + assert tbpl_status == results["tbpl_status"] + assert log_level == results["log_level"] crash = filter_action("crash", lines) - assert len(crash) == 0 + assert len(crash) == results["lines"] process_output = filter_action("process_output", lines) assert any("ERROR: AddressSanitizer" in l["data"] for l in process_output) @pytest.mark.skip_mozinfo("!debug") +@pytest.mark.parametrize("runFailures", [""]) @pytest.mark.parametrize("flavor", ["plain"]) -def test_output_assertion(flavor, runtests, test_name): - status, lines = runtests(test_name("assertion")) +def test_output_assertion(flavor, runFailures, runtests, test_name): + extra_opts = {} + results = { + "status": 0, + "tbpl_status": TBPL_WARNING, + "log_level": WARNING, + "lines": 1, + "assertions": 1, + } + + status, lines = runtests(test_name("assertion"), **extra_opts) # TODO: mochitest should return non-zero here - assert status == 0 + assert status == results["status"] tbpl_status, log_level, summary = get_mozharness_status(lines, status) - assert tbpl_status == TBPL_WARNING - assert log_level == WARNING + assert tbpl_status == results["tbpl_status"] + assert log_level == results["log_level"] test_end = filter_action("test_end", lines) - assert len(test_end) == 1 + assert len(test_end) == results["lines"] # TODO: this should be ASSERT, but moving the assertion check before # the test_end action caused a bunch of failures. assert test_end[0]["status"] == "OK" assertions = filter_action("assertion_count", lines) - assert len(assertions) == 1 - assert assertions[0]["count"] == 1 + assert len(assertions) == results["assertions"] + assert assertions[0]["count"] == results["assertions"] @pytest.mark.skip_mozinfo("!debug") +@pytest.mark.parametrize("runFailures", [""]) @pytest.mark.parametrize("flavor", ["plain"]) -def test_output_leak(flavor, runtests, test_name): - status, lines = runtests(test_name("leak")) +def test_output_leak(flavor, runFailures, runtests, test_name): + extra_opts = {} + results = {"status": 0, "tbpl_status": TBPL_WARNING, "log_level": WARNING} + + status, lines = runtests(test_name("leak"), **extra_opts) # TODO: mochitest should return non-zero here - assert status == 0 + assert status == results["status"] tbpl_status, log_level, summary = get_mozharness_status(lines, status) - assert tbpl_status == TBPL_WARNING - assert log_level == WARNING + assert tbpl_status == results["tbpl_status"] + assert log_level == results["log_level"] leak_totals = filter_action("mozleak_total", lines) found_leaks = False