diff --git a/utils/lit/lit/Test.py b/utils/lit/lit/Test.py index 38bb41b0252..701335541fb 100644 --- a/utils/lit/lit/Test.py +++ b/utils/lit/lit/Test.py @@ -27,6 +27,7 @@ class ResultCode(object): (self.name, self.isFailure)) PASS = ResultCode('PASS', False) +FLAKYPASS = ResultCode('FLAKYPASS', False) XFAIL = ResultCode('XFAIL', False) FAIL = ResultCode('FAIL', True) XPASS = ResultCode('XPASS', True) @@ -253,4 +254,4 @@ class Test: xml += "\n\t\n" else: xml += "/>" - return xml \ No newline at end of file + return xml diff --git a/utils/lit/lit/TestRunner.py b/utils/lit/lit/TestRunner.py index 055a282f3bc..24075ff9404 100644 --- a/utils/lit/lit/TestRunner.py +++ b/utils/lit/lit/TestRunner.py @@ -602,5 +602,17 @@ def executeShTest(test, litConfig, useExternalSh, return lit.Test.Result(Test.PASS) script, tmpBase, execdir = res - return _runShTest(test, litConfig, useExternalSh, script, tmpBase, execdir) + # Re-run failed tests up to test_retry_attempts times. + attempts = 1 + if hasattr(test.config, 'test_retry_attempts'): + attempts += test.config.test_retry_attempts + for i in range(attempts): + res = _runShTest(test, litConfig, useExternalSh, script, tmpBase, execdir) + if res.code != Test.FAIL: + break + # If we had to run the test more than once, count it as a flaky pass. These + # will be printed separately in the test summary. + if i > 0 and res.code == Test.PASS: + res.code = Test.FLAKYPASS + return res diff --git a/utils/lit/lit/main.py b/utils/lit/lit/main.py index e3722674f63..630cb54f96d 100755 --- a/utils/lit/lit/main.py +++ b/utils/lit/lit/main.py @@ -414,6 +414,7 @@ def main(builtinParameters = {}): lit.util.printHistogram(test_times, title='Tests') for name,code in (('Expected Passes ', lit.Test.PASS), + ('Passes With Retry ', lit.Test.FLAKYPASS), ('Expected Failures ', lit.Test.XFAIL), ('Unsupported Tests ', lit.Test.UNSUPPORTED), ('Unresolved Tests ', lit.Test.UNRESOLVED),