Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-19 00:05:36 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1699456 - When browser-chrome tests have a single test and crash, failures in log processing as the crash is handled in python instead of javascript. r=ahal

Browse Source 
Differential Revision: https://phabricator.services.mozilla.com/D108958
This commit is contained in:
Joel Maher 2021-03-31 22:09:29 +00:00
parent c8566ba9c7
commit 224db8c460
5 changed files with 177 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
testing/mochitest/browser-test.js
View File

@ -1259,7 +1259,10 @@ function testResult({ name, pass, todo, ex, stack, allowFailure }) {
if (allowFailure && !pass) {
this.allowedFailure = true;
this.pass = true;
this.todo = true;
this.todo = false;
} else if (allowFailure && pass) {
this.pass = true;
this.todo = false;
} else {
this.pass = !!pass;
this.todo = todo;

65
testing/mochitest/runtests.py
View File

@ -2549,16 +2549,26 @@ toolbar#nav-bar {
# record post-test information
if status:
self.message_logger.dump_buffered()
if crashAsPass:
self.log.info(
"TEST-PASS | %s | application terminated with exit code %s"
% (self.lastTestSeen, status)
)
msg = ("application terminated with exit code %s" % status,)
# self.message_logger.is_test_running indicates we need to send a test_end
if crashAsPass and self.message_logger.is_test_running:
# this works for browser-chrome, mochitest-plain has status=0
message = {
"action": "test_end",
"status": "CRASH",
"expected": "CRASH",
"thread": None,
"pid": None,
"source": "mochitest",
"time": int(time.time()) * 1000,
"test": self.lastTestSeen,
"message": msg,
}
# need to send a test_end in order to have mozharness process messages properly
# this requires a custom message vs log.error/log.warning/etc.
self.message_logger.process_message(message)
else:
self.log.error(
"TEST-UNEXPECTED-FAIL | %s | application terminated with exit code %s"
% (self.lastTestSeen, status)
)
self.log.error(msg)
else:
self.lastTestSeen = "Main app process exited normally"
@ -2576,6 +2586,7 @@ toolbar#nav-bar {
quiet = False
if crashAsPass:
quiet = True
minidump_path = os.path.join(self.profile.profile, "minidumps")
crash_count = mozcrash.log_crashes(
self.log,
@ -2585,12 +2596,27 @@ toolbar#nav-bar {
quiet=quiet,
)
if crash_count or zombieProcesses:
status = 1
if crashAsPass:
# self.message_logger.is_test_running indicates we need a test_end message
if crash_count > 0 and self.message_logger.is_test_running:
# this works for browser-chrome, mochitest-plain has status=0
message = {
"action": "test_end",
"status": "CRASH",
"expected": "CRASH",
"thread": None,
"pid": None,
"source": "mochitest",
"time": int(time.time()) * 1000,
"test": self.lastTestSeen,
"message": "application terminated with exit code 0",
}
# need to send a test_end in order to have mozharness process messages properly
# this requires a custom message vs log.error/log.warning/etc.
self.message_logger.process_message(message)
status = 0
elif crash_count or zombieProcesses:
status = 1
finally:
# cleanup
if os.path.exists(processLog):
@ -2934,6 +2960,14 @@ toolbar#nav-bar {
e10s_mode = "e10s" if options.e10s else "non-e10s"
# for failure mode: where browser window has crashed and we have no reported results
if (
self.countpass == self.countfail == self.counttodo == 0
and options.crashAsPass
):
self.countpass = 1
self.result = 0
# printing total number of tests
if options.flavor == "browser":
print("TEST-INFO | checking window state")
@ -3101,6 +3135,7 @@ toolbar#nav-bar {
mozinfo.info["debug"]
and options.flavor == "browser"
and options.subsuite != "thunderbird"
and not options.crashAsPass
)
self.start_script_kwargs["flavor"] = self.normflavor(options.flavor)
@ -3201,6 +3236,10 @@ toolbar#nav-bar {
ignoreMissingLeaks = options.ignoreMissingLeaks
leakThresholds = options.leakThresholds
if options.crashAsPass:
ignoreMissingLeaks.append("tab")
ignoreMissingLeaks.append("socket")
# Stop leak detection if m-bc code coverage is enabled
# by maxing out the leak threshold for all processes.
if options.jscov_dir_prefix:

2
testing/mochitest/tests/SimpleTest/SimpleTest.js
View File

@ -425,7 +425,7 @@ SimpleTest.record = function(condition, name, diag, stack, expected) {
if (SimpleTest.expected == "fail") {
if (!test.result) {
SimpleTest.num_failed++;
test.todo = true;
test.result = true;
}
successInfo = {
status: "PASS",

11
testing/mochitest/tests/python/conftest.py
View File

@ -38,7 +38,12 @@ def runtests(setup_test_harness, binary, parser, request):
if "flavor" in request.fixturenames:
flavor = request.getfixturevalue("flavor")
runFailures = ""
if "runFailures" in request.fixturenames:
runFailures = request.getfixturevalue("runFailures")
setup_test_harness(*setup_args, flavor=flavor)
runtests = pytest.importorskip("runtests")
mochitest_root = runtests.SCRIPT_DIR
@ -58,11 +63,16 @@ def runtests(setup_test_harness, binary, parser, request):
{
"app": binary,
"flavor": flavor,
"runFailures": runFailures,
"keep_open": False,
"log_raw": [buf],
}
)
if runFailures == "selftest":
options["crashAsPass"] = True
options["timeoutAsPass"] = True
if not os.path.isdir(runtests.build_obj.bindir):
package_root = os.path.dirname(mochitest_root)
options.update(
@ -85,6 +95,7 @@ def runtests(setup_test_harness, binary, parser, request):
# add a dummy manifest file because mochitest expects it
"manifest": os.path.join(test_root, manifest_name),
"manifest_relpath": manifest_name,
"skip-if": runFailures,
}
def inner(*tests, **opts):

154
testing/mochitest/tests/python/test_mochitest_integration.py
View File

@ -32,108 +32,172 @@ def test_name(request):
return inner
@pytest.mark.parametrize("runFailures", ["selftest", ""])
@pytest.mark.parametrize("flavor", ["plain", "browser-chrome"])
def test_output_pass(flavor, runtests, test_name):
status, lines = runtests(test_name("pass"))
assert status == 0
def test_output_pass(flavor, runFailures, runtests, test_name):
extra_opts = {}
results = {
"status": 1 if runFailures else 0,
"tbpl_status": TBPL_WARNING if runFailures else TBPL_SUCCESS,
"log_level": (INFO, WARNING),
"lines": 2 if runFailures else 1,
"line_status": "PASS",
}
if runFailures:
extra_opts["runFailures"] = runFailures
extra_opts["crashAsPass"] = True
extra_opts["timeoutAsPass"] = True
status, lines = runtests(test_name("pass"), **extra_opts)
assert status == results["status"]
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_SUCCESS
assert log_level in (INFO, WARNING)
assert tbpl_status == results["tbpl_status"]
assert log_level in results["log_level"]
lines = filter_action("test_status", lines)
assert len(lines) == 1
assert lines[0]["status"] == "PASS"
assert len(lines) == results["lines"]
assert lines[0]["status"] == results["line_status"]
@pytest.mark.parametrize("runFailures", ["selftest", ""])
@pytest.mark.parametrize("flavor", ["plain", "browser-chrome"])
def test_output_fail(flavor, runtests, test_name):
status, lines = runtests(test_name("fail"))
assert status == 1
def test_output_fail(flavor, runFailures, runtests, test_name):
extra_opts = {}
results = {
"status": 0 if runFailures else 1,
"tbpl_status": TBPL_SUCCESS if runFailures else TBPL_WARNING,
"log_level": (INFO, WARNING),
"lines": 1,
"line_status": "PASS" if runFailures else "FAIL",
}
if runFailures:
extra_opts["runFailures"] = runFailures
extra_opts["crashAsPass"] = True
extra_opts["timeoutAsPass"] = True
status, lines = runtests(test_name("fail"), **extra_opts)
assert status == results["status"]
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_WARNING
assert log_level == WARNING
assert tbpl_status == results["tbpl_status"]
assert log_level in results["log_level"]
lines = filter_action("test_status", lines)
assert len(lines) == 1
assert lines[0]["status"] == "FAIL"
assert len(lines) == results["lines"]
assert lines[0]["status"] == results["line_status"]
@pytest.mark.skip_mozinfo("!crashreporter")
@pytest.mark.parametrize("runFailures", ["selftest", ""])
@pytest.mark.parametrize("flavor", ["plain", "browser-chrome"])
def test_output_crash(flavor, runtests, test_name):
def test_output_crash(flavor, runFailures, runtests, test_name):
extra_opts = {}
results = {
"status": 0 if runFailures else 1,
"tbpl_status": TBPL_FAILURE,
"log_level": ERROR,
"lines": 1 if runFailures else 0,
}
if runFailures:
extra_opts["runFailures"] = runFailures
extra_opts["crashAsPass"] = True
extra_opts["timeoutAsPass"] = True
# bug 1443327 - we do not set MOZ_CRASHREPORTER_SHUTDOWN for browser-chrome
# the error regex's don't pick this up as a failure
if flavor == "browser-chrome":
results["tbpl_status"] = TBPL_SUCCESS
results["log_level"] = (INFO, WARNING)
status, lines = runtests(
test_name("crash"), environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"]
test_name("crash"), environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"], **extra_opts
)
assert status == 1
assert status == results["status"]
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_FAILURE
assert log_level == ERROR
assert tbpl_status == results["tbpl_status"]
assert log_level in results["log_level"]
crash = filter_action("crash", lines)
assert len(crash) == 1
assert crash[0]["action"] == "crash"
assert crash[0]["signature"]
assert crash[0]["minidump_path"]
if not runFailures:
crash = filter_action("crash", lines)
assert len(crash) == 1
assert crash[0]["action"] == "crash"
assert crash[0]["signature"]
assert crash[0]["minidump_path"]
lines = filter_action("test_end", lines)
assert len(lines) == 0
assert len(lines) == results["lines"]
@pytest.mark.skip_mozinfo("!asan")
@pytest.mark.parametrize("runFailures", [""])
@pytest.mark.parametrize("flavor", ["plain"])
def test_output_asan(flavor, runtests, test_name):
def test_output_asan(flavor, runFailures, runtests, test_name):
extra_opts = {}
results = {"status": 1, "tbpl_status": TBPL_FAILURE, "log_level": ERROR, "lines": 0}
status, lines = runtests(
test_name("crash"), environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"]
test_name("crash"), environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"], **extra_opts
)
assert status == 1
assert status == results["status"]
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_FAILURE
assert log_level == ERROR
assert tbpl_status == results["tbpl_status"]
assert log_level == results["log_level"]
crash = filter_action("crash", lines)
assert len(crash) == 0
assert len(crash) == results["lines"]
process_output = filter_action("process_output", lines)
assert any("ERROR: AddressSanitizer" in l["data"] for l in process_output)
@pytest.mark.skip_mozinfo("!debug")
@pytest.mark.parametrize("runFailures", [""])
@pytest.mark.parametrize("flavor", ["plain"])
def test_output_assertion(flavor, runtests, test_name):
status, lines = runtests(test_name("assertion"))
def test_output_assertion(flavor, runFailures, runtests, test_name):
extra_opts = {}
results = {
"status": 0,
"tbpl_status": TBPL_WARNING,
"log_level": WARNING,
"lines": 1,
"assertions": 1,
}
status, lines = runtests(test_name("assertion"), **extra_opts)
# TODO: mochitest should return non-zero here
assert status == 0
assert status == results["status"]
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_WARNING
assert log_level == WARNING
assert tbpl_status == results["tbpl_status"]
assert log_level == results["log_level"]
test_end = filter_action("test_end", lines)
assert len(test_end) == 1
assert len(test_end) == results["lines"]
# TODO: this should be ASSERT, but moving the assertion check before
# the test_end action caused a bunch of failures.
assert test_end[0]["status"] == "OK"
assertions = filter_action("assertion_count", lines)
assert len(assertions) == 1
assert assertions[0]["count"] == 1
assert len(assertions) == results["assertions"]
assert assertions[0]["count"] == results["assertions"]
@pytest.mark.skip_mozinfo("!debug")
@pytest.mark.parametrize("runFailures", [""])
@pytest.mark.parametrize("flavor", ["plain"])
def test_output_leak(flavor, runtests, test_name):
status, lines = runtests(test_name("leak"))
def test_output_leak(flavor, runFailures, runtests, test_name):
extra_opts = {}
results = {"status": 0, "tbpl_status": TBPL_WARNING, "log_level": WARNING}
status, lines = runtests(test_name("leak"), **extra_opts)
# TODO: mochitest should return non-zero here
assert status == 0
assert status == results["status"]
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
assert tbpl_status == TBPL_WARNING
assert log_level == WARNING
assert tbpl_status == results["tbpl_status"]
assert log_level == results["log_level"]
leak_totals = filter_action("mozleak_total", lines)
found_leaks = False