From 82a345196945e2da4bc73ed687e50565b40e98de Mon Sep 17 00:00:00 2001 From: Jeff Walden Date: Tue, 13 Jul 2010 18:04:29 -0500 Subject: [PATCH] Bug 571062 - Add slow-test annotation functionality to reftest and the JS test harness. r=dbaron, r=dmandelin --- js/src/tests/ecma/Date/jstests.list | 16 ++++----- js/src/tests/jstests.py | 5 +++ js/src/tests/manifest.py | 6 +++- js/src/tests/tests.py | 5 ++- layout/tools/reftest/README.txt | 11 +++++++ layout/tools/reftest/reftest-cmdline.js | 7 ++++ layout/tools/reftest/reftest.js | 44 ++++++++++++++++++++----- layout/tools/reftest/runreftest.py | 5 +++ 8 files changed, 81 insertions(+), 18 deletions(-) diff --git a/js/src/tests/ecma/Date/jstests.list b/js/src/tests/ecma/Date/jstests.list index 443656aabedc..ecb6b2f10e26 100644 --- a/js/src/tests/ecma/Date/jstests.list +++ b/js/src/tests/ecma/Date/jstests.list @@ -149,11 +149,11 @@ script 15.9.5.7.js script 15.9.5.8.js script 15.9.5.9.js script 15.9.5.js -script dst-offset-caching-1-of-8.js -script dst-offset-caching-2-of-8.js -script dst-offset-caching-3-of-8.js -script dst-offset-caching-4-of-8.js -script dst-offset-caching-5-of-8.js -script dst-offset-caching-6-of-8.js -script dst-offset-caching-7-of-8.js -script dst-offset-caching-8-of-8.js +slow script dst-offset-caching-1-of-8.js +slow script dst-offset-caching-2-of-8.js +slow script dst-offset-caching-3-of-8.js +slow script dst-offset-caching-4-of-8.js +slow script dst-offset-caching-5-of-8.js +slow script dst-offset-caching-6-of-8.js +slow script dst-offset-caching-7-of-8.js +slow script dst-offset-caching-8-of-8.js diff --git a/js/src/tests/jstests.py b/js/src/tests/jstests.py index a8a3819587d6..dafb5dd13b1a 100644 --- a/js/src/tests/jstests.py +++ b/js/src/tests/jstests.py @@ -227,6 +227,8 @@ if __name__ == '__main__': help='check for test files not listed in the manifest') op.add_option('--failure-file', dest='failure_file', help='write tests that have not passed to the given file') + op.add_option('--run-slow-tests', dest='run_slow_tests', action='store_true', + help='run particularly slow tests as well as average-speed tests') (OPTIONS, args) = op.parse_args() if len(args) < 1: if not OPTIONS.check_manifest: @@ -310,6 +312,9 @@ if __name__ == '__main__': OPTIONS.run_skipped = True test_list = [ _ for _ in test_list if not _.enable ] + if not OPTIONS.run_slow_tests: + test_list = [ _ for _ in test_list if not _.slow ] + if OPTIONS.debug and test_list: if len(test_list) > 1: print('Multiple tests match command line arguments, debugger can only run one') diff --git a/js/src/tests/manifest.py b/js/src/tests/manifest.py index f904c0b3dc65..1b4f8b7ada9e 100644 --- a/js/src/tests/manifest.py +++ b/js/src/tests/manifest.py @@ -110,6 +110,7 @@ def parse(filename, xul_tester, reldir = ''): enable = True expect = True random = False + slow = False pos = 0 while pos < len(parts): @@ -144,11 +145,14 @@ def parse(filename, xul_tester, reldir = ''): elif parts[pos] == 'script': script = parts[pos+1] pos += 2 + elif parts[pos] == 'slow': + slow = True + pos += 1 else: print 'warning: invalid manifest line element "%s"'%parts[pos] pos += 1 assert script is not None ans.append(TestCase(os.path.join(reldir, script), - enable, expect, random)) + enable, expect, random, slow)) return ans diff --git a/js/src/tests/tests.py b/js/src/tests/tests.py index 6e0966c92071..6405531160e2 100644 --- a/js/src/tests/tests.py +++ b/js/src/tests/tests.py @@ -87,11 +87,12 @@ class Test(object): class TestCase(Test): """A test case consisting of a test and an expected result.""" - def __init__(self, path, enable, expect, random): + def __init__(self, path, enable, expect, random, slow): Test.__init__(self, path) self.enable = enable # bool: True => run test, False => don't run self.expect = expect # bool: expected result, True => pass self.random = random # bool: True => ignore output as 'random' + self.slow = slow # bool: True => test may run slowly def __str__(self): ans = self.path @@ -101,6 +102,8 @@ class TestCase(Test): ans += ', fails' if self.random: ans += ', random' + if self.slow: + ans += ', slow' return ans class TestOutput: diff --git a/layout/tools/reftest/README.txt b/layout/tools/reftest/README.txt index 0e76133121ec..abcbb80ae4e3 100644 --- a/layout/tools/reftest/README.txt +++ b/layout/tools/reftest/README.txt @@ -74,6 +74,17 @@ must be one of the following: particular platform (i.e. it allows us to get test coverage on the other platforms). + slow The test may take a long time to run, so run it if slow tests are + either enabled or not disabled (test manifest interpreters may + choose whether or not to run such tests by default). + + slow-if(condition) If the condition is met, the test is treated as if + 'slow' had been specified. This is useful for tests + which are slow only on particular platforms (e.g. a + test which exercised out-of-memory behavior might be + fast on a 32-bit system but inordinately slow on a + 64-bit system). + asserts(count) Loading the test and reference is known to assert exactly count times. diff --git a/layout/tools/reftest/reftest-cmdline.js b/layout/tools/reftest/reftest-cmdline.js index d8e956dd5b7d..3f10d2ff9353 100644 --- a/layout/tools/reftest/reftest-cmdline.js +++ b/layout/tools/reftest/reftest-cmdline.js @@ -79,6 +79,13 @@ RefTestCmdLineHandler.prototype = catch (e) { } + try { + var skipslowtests = cmdLine.handleFlag("reftestskipslowtests", false); + args.skipslowtests = skipslowtests; + } + catch (e) { + } + /* Ignore the platform's online/offline status while running reftests. */ var ios = Components.classes["@mozilla.org/network/io-service;1"] .getService(Components.interfaces.nsIIOService2); diff --git a/layout/tools/reftest/reftest.js b/layout/tools/reftest/reftest.js index 07f96f484ba3..475d6cf10730 100644 --- a/layout/tools/reftest/reftest.js +++ b/layout/tools/reftest/reftest.js @@ -90,6 +90,7 @@ var gTestResults = { AssertionKnown: 0, Random : 0, Skip: 0, + Slow: 0, }; var gTotalTests = 0; var gState; @@ -127,6 +128,9 @@ const gProtocolRE = /^\w+:/; var HTTP_SERVER_PORT = 4444; const HTTP_SERVER_PORTS_TO_TRY = 50; +// whether to run slow tests or not +var gRunSlowTests = true; + // whether we should skip caching canvases var gNoCanvasCache = false; @@ -259,6 +263,9 @@ function StartTests() if ("nocache" in args && args["nocache"]) gNoCanvasCache = true; + if ("skipslowtests" in args && args.skipslowtests) + gRunSlowTests = false; + ReadTopManifest(args.uri); BuildUseCounts(); @@ -444,7 +451,8 @@ function ReadManifest(aURL) var expected_status = EXPECTED_PASS; var minAsserts = 0; var maxAsserts = 0; - while (items[0].match(/^(fails|random|skip|asserts)/)) { + var slow = false; + while (items[0].match(/^(fails|random|skip|asserts|slow)/)) { var item = items.shift(); var stat; var cond; @@ -469,6 +477,13 @@ function ReadManifest(aURL) (m[3] == undefined) ? minAsserts : Number(m[3].substring(1)); } + } else if (item == "slow") { + cond = false; + slow = true; + } else if ((m = item.match(/^slow-if\((.*?)\)$/))) { + cond = false; + if (Components.utils.evalInSandbox("(" + m[1] + ")", sandbox)) + slow = true; } else { throw "Error 1 in manifest file " + aURL.spec + " line " + lineNo; } @@ -540,6 +555,7 @@ function ReadManifest(aURL) prettyPath: prettyPath, minAsserts: minAsserts, maxAsserts: maxAsserts, + slow: slow, url1: testURI, url2: null } ); } else if (items[0] == TYPE_SCRIPT) { @@ -559,6 +575,7 @@ function ReadManifest(aURL) prettyPath: prettyPath, minAsserts: minAsserts, maxAsserts: maxAsserts, + slow: slow, url1: testURI, url2: null } ); } else if (items[0] == TYPE_REFTEST_EQUAL || items[0] == TYPE_REFTEST_NOTEQUAL) { @@ -581,6 +598,7 @@ function ReadManifest(aURL) prettyPath: prettyPath, minAsserts: minAsserts, maxAsserts: maxAsserts, + slow: slow, url1: testURI, url2: refURI } ); } else { @@ -660,10 +678,19 @@ function ServeFiles(manifestURL, depth, aURL, files) function StartCurrentTest() { // make sure we don't run tests that are expected to kill the browser - while (gURLs.length > 0 && gURLs[0].expected == EXPECTED_DEATH) { - ++gTestResults.Skip; - gDumpLog("REFTEST TEST-KNOWN-FAIL | " + gURLs[0].url1.spec + " | (SKIP)\n"); - gURLs.shift(); + while (gURLs.length > 0) { + var test = gURLs[0]; + if (test.expected == EXPECTED_DEATH) { + ++gTestResults.Skip; + gDumpLog("REFTEST TEST-KNOWN-FAIL | " + test.url1.spec + " | (SKIP)\n"); + gURLs.shift(); + } else if (test.slow && !gRunSlowTests) { + ++gTestResults.Slow; + gDumpLog("REFTEST TEST-KNOWN-SLOW | " + test.url1.spec + " | (SLOW)\n"); + gURLs.shift(); + } else { + break; + } } if (gURLs.length == 0) { @@ -726,12 +753,13 @@ function DoneTests() gTestResults.FailedLoad + " failed load, " + gTestResults.Exception + " exception)\n"); count = gTestResults.KnownFail + gTestResults.AssertionKnown + - gTestResults.Random + gTestResults.Skip; - gDumpLog("REFTEST INFO | Known problems: " + count + " (" + + gTestResults.Random + gTestResults.Skip + gTestResults.Slow; + dump("REFTEST INFO | Known problems: " + count + " (" + gTestResults.KnownFail + " known fail, " + gTestResults.AssertionKnown + " known asserts, " + gTestResults.Random + " random, " + - gTestResults.Skip + " skipped)\n"); + gTestResults.Skip + " skipped, " + + gTestResults.Slow + " slow)\n"); gDumpLog("REFTEST INFO | Total canvas count = " + gRecycledCanvases.length + "\n"); diff --git a/layout/tools/reftest/runreftest.py b/layout/tools/reftest/runreftest.py index 322c6ade751a..d5bc50a765cf 100644 --- a/layout/tools/reftest/runreftest.py +++ b/layout/tools/reftest/runreftest.py @@ -229,6 +229,11 @@ class ReftestOptions(OptionParser): help = "file to log output to in addition to stdout") defaults["logFile"] = None + self.add_option("--skip-slow-tests", + dest = "skipSlowTests", action = "store_true", + help = "skip tests marked as slow when running") + defaults["skipSlowTests"] = False + self.set_defaults(**defaults) def main():