mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-10-19 00:05:36 +00:00
Bug 1699456 - When browser-chrome tests have a single test and crash, failures in log processing as the crash is handled in python instead of javascript. r=ahal
Differential Revision: https://phabricator.services.mozilla.com/D108958
This commit is contained in:
parent
c8566ba9c7
commit
224db8c460
@ -1259,7 +1259,10 @@ function testResult({ name, pass, todo, ex, stack, allowFailure }) {
|
|||||||
if (allowFailure && !pass) {
|
if (allowFailure && !pass) {
|
||||||
this.allowedFailure = true;
|
this.allowedFailure = true;
|
||||||
this.pass = true;
|
this.pass = true;
|
||||||
this.todo = true;
|
this.todo = false;
|
||||||
|
} else if (allowFailure && pass) {
|
||||||
|
this.pass = true;
|
||||||
|
this.todo = false;
|
||||||
} else {
|
} else {
|
||||||
this.pass = !!pass;
|
this.pass = !!pass;
|
||||||
this.todo = todo;
|
this.todo = todo;
|
||||||
|
@ -2549,16 +2549,26 @@ toolbar#nav-bar {
|
|||||||
# record post-test information
|
# record post-test information
|
||||||
if status:
|
if status:
|
||||||
self.message_logger.dump_buffered()
|
self.message_logger.dump_buffered()
|
||||||
if crashAsPass:
|
msg = ("application terminated with exit code %s" % status,)
|
||||||
self.log.info(
|
# self.message_logger.is_test_running indicates we need to send a test_end
|
||||||
"TEST-PASS | %s | application terminated with exit code %s"
|
if crashAsPass and self.message_logger.is_test_running:
|
||||||
% (self.lastTestSeen, status)
|
# this works for browser-chrome, mochitest-plain has status=0
|
||||||
)
|
message = {
|
||||||
|
"action": "test_end",
|
||||||
|
"status": "CRASH",
|
||||||
|
"expected": "CRASH",
|
||||||
|
"thread": None,
|
||||||
|
"pid": None,
|
||||||
|
"source": "mochitest",
|
||||||
|
"time": int(time.time()) * 1000,
|
||||||
|
"test": self.lastTestSeen,
|
||||||
|
"message": msg,
|
||||||
|
}
|
||||||
|
# need to send a test_end in order to have mozharness process messages properly
|
||||||
|
# this requires a custom message vs log.error/log.warning/etc.
|
||||||
|
self.message_logger.process_message(message)
|
||||||
else:
|
else:
|
||||||
self.log.error(
|
self.log.error(msg)
|
||||||
"TEST-UNEXPECTED-FAIL | %s | application terminated with exit code %s"
|
|
||||||
% (self.lastTestSeen, status)
|
|
||||||
)
|
|
||||||
else:
|
else:
|
||||||
self.lastTestSeen = "Main app process exited normally"
|
self.lastTestSeen = "Main app process exited normally"
|
||||||
|
|
||||||
@ -2576,6 +2586,7 @@ toolbar#nav-bar {
|
|||||||
quiet = False
|
quiet = False
|
||||||
if crashAsPass:
|
if crashAsPass:
|
||||||
quiet = True
|
quiet = True
|
||||||
|
|
||||||
minidump_path = os.path.join(self.profile.profile, "minidumps")
|
minidump_path = os.path.join(self.profile.profile, "minidumps")
|
||||||
crash_count = mozcrash.log_crashes(
|
crash_count = mozcrash.log_crashes(
|
||||||
self.log,
|
self.log,
|
||||||
@ -2585,12 +2596,27 @@ toolbar#nav-bar {
|
|||||||
quiet=quiet,
|
quiet=quiet,
|
||||||
)
|
)
|
||||||
|
|
||||||
if crash_count or zombieProcesses:
|
|
||||||
status = 1
|
|
||||||
|
|
||||||
if crashAsPass:
|
if crashAsPass:
|
||||||
|
# self.message_logger.is_test_running indicates we need a test_end message
|
||||||
|
if crash_count > 0 and self.message_logger.is_test_running:
|
||||||
|
# this works for browser-chrome, mochitest-plain has status=0
|
||||||
|
message = {
|
||||||
|
"action": "test_end",
|
||||||
|
"status": "CRASH",
|
||||||
|
"expected": "CRASH",
|
||||||
|
"thread": None,
|
||||||
|
"pid": None,
|
||||||
|
"source": "mochitest",
|
||||||
|
"time": int(time.time()) * 1000,
|
||||||
|
"test": self.lastTestSeen,
|
||||||
|
"message": "application terminated with exit code 0",
|
||||||
|
}
|
||||||
|
# need to send a test_end in order to have mozharness process messages properly
|
||||||
|
# this requires a custom message vs log.error/log.warning/etc.
|
||||||
|
self.message_logger.process_message(message)
|
||||||
status = 0
|
status = 0
|
||||||
|
elif crash_count or zombieProcesses:
|
||||||
|
status = 1
|
||||||
finally:
|
finally:
|
||||||
# cleanup
|
# cleanup
|
||||||
if os.path.exists(processLog):
|
if os.path.exists(processLog):
|
||||||
@ -2934,6 +2960,14 @@ toolbar#nav-bar {
|
|||||||
|
|
||||||
e10s_mode = "e10s" if options.e10s else "non-e10s"
|
e10s_mode = "e10s" if options.e10s else "non-e10s"
|
||||||
|
|
||||||
|
# for failure mode: where browser window has crashed and we have no reported results
|
||||||
|
if (
|
||||||
|
self.countpass == self.countfail == self.counttodo == 0
|
||||||
|
and options.crashAsPass
|
||||||
|
):
|
||||||
|
self.countpass = 1
|
||||||
|
self.result = 0
|
||||||
|
|
||||||
# printing total number of tests
|
# printing total number of tests
|
||||||
if options.flavor == "browser":
|
if options.flavor == "browser":
|
||||||
print("TEST-INFO | checking window state")
|
print("TEST-INFO | checking window state")
|
||||||
@ -3101,6 +3135,7 @@ toolbar#nav-bar {
|
|||||||
mozinfo.info["debug"]
|
mozinfo.info["debug"]
|
||||||
and options.flavor == "browser"
|
and options.flavor == "browser"
|
||||||
and options.subsuite != "thunderbird"
|
and options.subsuite != "thunderbird"
|
||||||
|
and not options.crashAsPass
|
||||||
)
|
)
|
||||||
|
|
||||||
self.start_script_kwargs["flavor"] = self.normflavor(options.flavor)
|
self.start_script_kwargs["flavor"] = self.normflavor(options.flavor)
|
||||||
@ -3201,6 +3236,10 @@ toolbar#nav-bar {
|
|||||||
ignoreMissingLeaks = options.ignoreMissingLeaks
|
ignoreMissingLeaks = options.ignoreMissingLeaks
|
||||||
leakThresholds = options.leakThresholds
|
leakThresholds = options.leakThresholds
|
||||||
|
|
||||||
|
if options.crashAsPass:
|
||||||
|
ignoreMissingLeaks.append("tab")
|
||||||
|
ignoreMissingLeaks.append("socket")
|
||||||
|
|
||||||
# Stop leak detection if m-bc code coverage is enabled
|
# Stop leak detection if m-bc code coverage is enabled
|
||||||
# by maxing out the leak threshold for all processes.
|
# by maxing out the leak threshold for all processes.
|
||||||
if options.jscov_dir_prefix:
|
if options.jscov_dir_prefix:
|
||||||
|
@ -425,7 +425,7 @@ SimpleTest.record = function(condition, name, diag, stack, expected) {
|
|||||||
if (SimpleTest.expected == "fail") {
|
if (SimpleTest.expected == "fail") {
|
||||||
if (!test.result) {
|
if (!test.result) {
|
||||||
SimpleTest.num_failed++;
|
SimpleTest.num_failed++;
|
||||||
test.todo = true;
|
test.result = true;
|
||||||
}
|
}
|
||||||
successInfo = {
|
successInfo = {
|
||||||
status: "PASS",
|
status: "PASS",
|
||||||
|
@ -38,7 +38,12 @@ def runtests(setup_test_harness, binary, parser, request):
|
|||||||
if "flavor" in request.fixturenames:
|
if "flavor" in request.fixturenames:
|
||||||
flavor = request.getfixturevalue("flavor")
|
flavor = request.getfixturevalue("flavor")
|
||||||
|
|
||||||
|
runFailures = ""
|
||||||
|
if "runFailures" in request.fixturenames:
|
||||||
|
runFailures = request.getfixturevalue("runFailures")
|
||||||
|
|
||||||
setup_test_harness(*setup_args, flavor=flavor)
|
setup_test_harness(*setup_args, flavor=flavor)
|
||||||
|
|
||||||
runtests = pytest.importorskip("runtests")
|
runtests = pytest.importorskip("runtests")
|
||||||
|
|
||||||
mochitest_root = runtests.SCRIPT_DIR
|
mochitest_root = runtests.SCRIPT_DIR
|
||||||
@ -58,11 +63,16 @@ def runtests(setup_test_harness, binary, parser, request):
|
|||||||
{
|
{
|
||||||
"app": binary,
|
"app": binary,
|
||||||
"flavor": flavor,
|
"flavor": flavor,
|
||||||
|
"runFailures": runFailures,
|
||||||
"keep_open": False,
|
"keep_open": False,
|
||||||
"log_raw": [buf],
|
"log_raw": [buf],
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if runFailures == "selftest":
|
||||||
|
options["crashAsPass"] = True
|
||||||
|
options["timeoutAsPass"] = True
|
||||||
|
|
||||||
if not os.path.isdir(runtests.build_obj.bindir):
|
if not os.path.isdir(runtests.build_obj.bindir):
|
||||||
package_root = os.path.dirname(mochitest_root)
|
package_root = os.path.dirname(mochitest_root)
|
||||||
options.update(
|
options.update(
|
||||||
@ -85,6 +95,7 @@ def runtests(setup_test_harness, binary, parser, request):
|
|||||||
# add a dummy manifest file because mochitest expects it
|
# add a dummy manifest file because mochitest expects it
|
||||||
"manifest": os.path.join(test_root, manifest_name),
|
"manifest": os.path.join(test_root, manifest_name),
|
||||||
"manifest_relpath": manifest_name,
|
"manifest_relpath": manifest_name,
|
||||||
|
"skip-if": runFailures,
|
||||||
}
|
}
|
||||||
|
|
||||||
def inner(*tests, **opts):
|
def inner(*tests, **opts):
|
||||||
|
@ -32,108 +32,172 @@ def test_name(request):
|
|||||||
return inner
|
return inner
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("runFailures", ["selftest", ""])
|
||||||
@pytest.mark.parametrize("flavor", ["plain", "browser-chrome"])
|
@pytest.mark.parametrize("flavor", ["plain", "browser-chrome"])
|
||||||
def test_output_pass(flavor, runtests, test_name):
|
def test_output_pass(flavor, runFailures, runtests, test_name):
|
||||||
status, lines = runtests(test_name("pass"))
|
extra_opts = {}
|
||||||
assert status == 0
|
results = {
|
||||||
|
"status": 1 if runFailures else 0,
|
||||||
|
"tbpl_status": TBPL_WARNING if runFailures else TBPL_SUCCESS,
|
||||||
|
"log_level": (INFO, WARNING),
|
||||||
|
"lines": 2 if runFailures else 1,
|
||||||
|
"line_status": "PASS",
|
||||||
|
}
|
||||||
|
if runFailures:
|
||||||
|
extra_opts["runFailures"] = runFailures
|
||||||
|
extra_opts["crashAsPass"] = True
|
||||||
|
extra_opts["timeoutAsPass"] = True
|
||||||
|
|
||||||
|
status, lines = runtests(test_name("pass"), **extra_opts)
|
||||||
|
assert status == results["status"]
|
||||||
|
|
||||||
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
|
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
|
||||||
assert tbpl_status == TBPL_SUCCESS
|
assert tbpl_status == results["tbpl_status"]
|
||||||
assert log_level in (INFO, WARNING)
|
assert log_level in results["log_level"]
|
||||||
|
|
||||||
lines = filter_action("test_status", lines)
|
lines = filter_action("test_status", lines)
|
||||||
assert len(lines) == 1
|
assert len(lines) == results["lines"]
|
||||||
assert lines[0]["status"] == "PASS"
|
assert lines[0]["status"] == results["line_status"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("runFailures", ["selftest", ""])
|
||||||
@pytest.mark.parametrize("flavor", ["plain", "browser-chrome"])
|
@pytest.mark.parametrize("flavor", ["plain", "browser-chrome"])
|
||||||
def test_output_fail(flavor, runtests, test_name):
|
def test_output_fail(flavor, runFailures, runtests, test_name):
|
||||||
status, lines = runtests(test_name("fail"))
|
extra_opts = {}
|
||||||
assert status == 1
|
results = {
|
||||||
|
"status": 0 if runFailures else 1,
|
||||||
|
"tbpl_status": TBPL_SUCCESS if runFailures else TBPL_WARNING,
|
||||||
|
"log_level": (INFO, WARNING),
|
||||||
|
"lines": 1,
|
||||||
|
"line_status": "PASS" if runFailures else "FAIL",
|
||||||
|
}
|
||||||
|
if runFailures:
|
||||||
|
extra_opts["runFailures"] = runFailures
|
||||||
|
extra_opts["crashAsPass"] = True
|
||||||
|
extra_opts["timeoutAsPass"] = True
|
||||||
|
|
||||||
|
status, lines = runtests(test_name("fail"), **extra_opts)
|
||||||
|
assert status == results["status"]
|
||||||
|
|
||||||
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
|
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
|
||||||
assert tbpl_status == TBPL_WARNING
|
assert tbpl_status == results["tbpl_status"]
|
||||||
assert log_level == WARNING
|
assert log_level in results["log_level"]
|
||||||
|
|
||||||
lines = filter_action("test_status", lines)
|
lines = filter_action("test_status", lines)
|
||||||
|
assert len(lines) == results["lines"]
|
||||||
assert len(lines) == 1
|
assert lines[0]["status"] == results["line_status"]
|
||||||
assert lines[0]["status"] == "FAIL"
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skip_mozinfo("!crashreporter")
|
@pytest.mark.skip_mozinfo("!crashreporter")
|
||||||
|
@pytest.mark.parametrize("runFailures", ["selftest", ""])
|
||||||
@pytest.mark.parametrize("flavor", ["plain", "browser-chrome"])
|
@pytest.mark.parametrize("flavor", ["plain", "browser-chrome"])
|
||||||
def test_output_crash(flavor, runtests, test_name):
|
def test_output_crash(flavor, runFailures, runtests, test_name):
|
||||||
|
extra_opts = {}
|
||||||
|
results = {
|
||||||
|
"status": 0 if runFailures else 1,
|
||||||
|
"tbpl_status": TBPL_FAILURE,
|
||||||
|
"log_level": ERROR,
|
||||||
|
"lines": 1 if runFailures else 0,
|
||||||
|
}
|
||||||
|
if runFailures:
|
||||||
|
extra_opts["runFailures"] = runFailures
|
||||||
|
extra_opts["crashAsPass"] = True
|
||||||
|
extra_opts["timeoutAsPass"] = True
|
||||||
|
# bug 1443327 - we do not set MOZ_CRASHREPORTER_SHUTDOWN for browser-chrome
|
||||||
|
# the error regex's don't pick this up as a failure
|
||||||
|
if flavor == "browser-chrome":
|
||||||
|
results["tbpl_status"] = TBPL_SUCCESS
|
||||||
|
results["log_level"] = (INFO, WARNING)
|
||||||
|
|
||||||
status, lines = runtests(
|
status, lines = runtests(
|
||||||
test_name("crash"), environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"]
|
test_name("crash"), environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"], **extra_opts
|
||||||
)
|
)
|
||||||
assert status == 1
|
assert status == results["status"]
|
||||||
|
|
||||||
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
|
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
|
||||||
assert tbpl_status == TBPL_FAILURE
|
assert tbpl_status == results["tbpl_status"]
|
||||||
assert log_level == ERROR
|
assert log_level in results["log_level"]
|
||||||
|
|
||||||
crash = filter_action("crash", lines)
|
if not runFailures:
|
||||||
assert len(crash) == 1
|
crash = filter_action("crash", lines)
|
||||||
assert crash[0]["action"] == "crash"
|
assert len(crash) == 1
|
||||||
assert crash[0]["signature"]
|
assert crash[0]["action"] == "crash"
|
||||||
assert crash[0]["minidump_path"]
|
assert crash[0]["signature"]
|
||||||
|
assert crash[0]["minidump_path"]
|
||||||
|
|
||||||
lines = filter_action("test_end", lines)
|
lines = filter_action("test_end", lines)
|
||||||
assert len(lines) == 0
|
assert len(lines) == results["lines"]
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skip_mozinfo("!asan")
|
@pytest.mark.skip_mozinfo("!asan")
|
||||||
|
@pytest.mark.parametrize("runFailures", [""])
|
||||||
@pytest.mark.parametrize("flavor", ["plain"])
|
@pytest.mark.parametrize("flavor", ["plain"])
|
||||||
def test_output_asan(flavor, runtests, test_name):
|
def test_output_asan(flavor, runFailures, runtests, test_name):
|
||||||
|
extra_opts = {}
|
||||||
|
results = {"status": 1, "tbpl_status": TBPL_FAILURE, "log_level": ERROR, "lines": 0}
|
||||||
|
|
||||||
status, lines = runtests(
|
status, lines = runtests(
|
||||||
test_name("crash"), environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"]
|
test_name("crash"), environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"], **extra_opts
|
||||||
)
|
)
|
||||||
assert status == 1
|
assert status == results["status"]
|
||||||
|
|
||||||
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
|
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
|
||||||
assert tbpl_status == TBPL_FAILURE
|
assert tbpl_status == results["tbpl_status"]
|
||||||
assert log_level == ERROR
|
assert log_level == results["log_level"]
|
||||||
|
|
||||||
crash = filter_action("crash", lines)
|
crash = filter_action("crash", lines)
|
||||||
assert len(crash) == 0
|
assert len(crash) == results["lines"]
|
||||||
|
|
||||||
process_output = filter_action("process_output", lines)
|
process_output = filter_action("process_output", lines)
|
||||||
assert any("ERROR: AddressSanitizer" in l["data"] for l in process_output)
|
assert any("ERROR: AddressSanitizer" in l["data"] for l in process_output)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skip_mozinfo("!debug")
|
@pytest.mark.skip_mozinfo("!debug")
|
||||||
|
@pytest.mark.parametrize("runFailures", [""])
|
||||||
@pytest.mark.parametrize("flavor", ["plain"])
|
@pytest.mark.parametrize("flavor", ["plain"])
|
||||||
def test_output_assertion(flavor, runtests, test_name):
|
def test_output_assertion(flavor, runFailures, runtests, test_name):
|
||||||
status, lines = runtests(test_name("assertion"))
|
extra_opts = {}
|
||||||
|
results = {
|
||||||
|
"status": 0,
|
||||||
|
"tbpl_status": TBPL_WARNING,
|
||||||
|
"log_level": WARNING,
|
||||||
|
"lines": 1,
|
||||||
|
"assertions": 1,
|
||||||
|
}
|
||||||
|
|
||||||
|
status, lines = runtests(test_name("assertion"), **extra_opts)
|
||||||
# TODO: mochitest should return non-zero here
|
# TODO: mochitest should return non-zero here
|
||||||
assert status == 0
|
assert status == results["status"]
|
||||||
|
|
||||||
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
|
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
|
||||||
assert tbpl_status == TBPL_WARNING
|
assert tbpl_status == results["tbpl_status"]
|
||||||
assert log_level == WARNING
|
assert log_level == results["log_level"]
|
||||||
|
|
||||||
test_end = filter_action("test_end", lines)
|
test_end = filter_action("test_end", lines)
|
||||||
assert len(test_end) == 1
|
assert len(test_end) == results["lines"]
|
||||||
# TODO: this should be ASSERT, but moving the assertion check before
|
# TODO: this should be ASSERT, but moving the assertion check before
|
||||||
# the test_end action caused a bunch of failures.
|
# the test_end action caused a bunch of failures.
|
||||||
assert test_end[0]["status"] == "OK"
|
assert test_end[0]["status"] == "OK"
|
||||||
|
|
||||||
assertions = filter_action("assertion_count", lines)
|
assertions = filter_action("assertion_count", lines)
|
||||||
assert len(assertions) == 1
|
assert len(assertions) == results["assertions"]
|
||||||
assert assertions[0]["count"] == 1
|
assert assertions[0]["count"] == results["assertions"]
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skip_mozinfo("!debug")
|
@pytest.mark.skip_mozinfo("!debug")
|
||||||
|
@pytest.mark.parametrize("runFailures", [""])
|
||||||
@pytest.mark.parametrize("flavor", ["plain"])
|
@pytest.mark.parametrize("flavor", ["plain"])
|
||||||
def test_output_leak(flavor, runtests, test_name):
|
def test_output_leak(flavor, runFailures, runtests, test_name):
|
||||||
status, lines = runtests(test_name("leak"))
|
extra_opts = {}
|
||||||
|
results = {"status": 0, "tbpl_status": TBPL_WARNING, "log_level": WARNING}
|
||||||
|
|
||||||
|
status, lines = runtests(test_name("leak"), **extra_opts)
|
||||||
# TODO: mochitest should return non-zero here
|
# TODO: mochitest should return non-zero here
|
||||||
assert status == 0
|
assert status == results["status"]
|
||||||
|
|
||||||
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
|
tbpl_status, log_level, summary = get_mozharness_status(lines, status)
|
||||||
assert tbpl_status == TBPL_WARNING
|
assert tbpl_status == results["tbpl_status"]
|
||||||
assert log_level == WARNING
|
assert log_level == results["log_level"]
|
||||||
|
|
||||||
leak_totals = filter_action("mozleak_total", lines)
|
leak_totals = filter_action("mozleak_total", lines)
|
||||||
found_leaks = False
|
found_leaks = False
|
||||||
|
Loading…
Reference in New Issue
Block a user