From 028c09511bd506471368d5c7bd35e0475c7059f2 Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Fri, 22 Mar 2019 12:56:42 +0100 Subject: [PATCH] pkg/bisect: detect when too many instances errored We currently skip a commit iff all 10 instances errored. But if, say, only 9 errored we consider it as OK, but this significnalty reduces chances of detecting flaky crashes. So skip if more than 2/3 errored. Update #501 --- pkg/bisect/bisect.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/bisect/bisect.go b/pkg/bisect/bisect.go index eff0601b..dea41581 100644 --- a/pkg/bisect/bisect.go +++ b/pkg/bisect/bisect.go @@ -275,7 +275,8 @@ func (env *env) test() (vcs.BisectResult, *vcs.Commit, *report.Report, error) { return vcs.BisectSkip, current, nil, nil } testStart := time.Now() - results, err := env.inst.Test(10, cfg.Repro.Syz, cfg.Repro.Opts, cfg.Repro.C) + const numTests = 10 + results, err := env.inst.Test(numTests, cfg.Repro.Syz, cfg.Repro.Opts, cfg.Repro.C) env.testTime += time.Since(testStart) if err != nil { env.log("failed: %v", err) @@ -285,6 +286,10 @@ func (env *env) test() (vcs.BisectResult, *vcs.Commit, *report.Report, error) { res := vcs.BisectSkip if bad != 0 { res = vcs.BisectBad + } else if numTests-good-bad > numTests/3*2 { + // More than 2/3 of instances failed with infrastructure error, + // can't reliably tell that the commit is good. + res = vcs.BisectSkip } else if good != 0 { res = vcs.BisectGood }