From b06a63128a50a11cb6877f76c28b8b824a4473e9 Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Thu, 1 Sep 2016 17:08:22 +0200 Subject: [PATCH] manager: improve how crashes are saved Now crashes dir contains 1 subdirectory per unique crash type. Each subdirectory contains 'description' file with a unique string identifying the crash type (e.g. "KASAN: slab-out-of-bounds Read of size 2 in bit_putcs"), and up to 100 logN and reportN files with raw crash log (as before) and post processed kernel oops message. --- report/report.go | 16 +++---- syz-manager/manager.go | 101 +++++++++++++++++++++++------------------ 2 files changed, 63 insertions(+), 54 deletions(-) diff --git a/report/report.go b/report/report.go index c946cc55..77728b07 100644 --- a/report/report.go +++ b/report/report.go @@ -186,9 +186,8 @@ func ContainsCrash(output []byte) bool { // Desc contains a representative description of the first oops (empty if no oops found), // text contains whole oops text, // start and end denote region of output with oops message(s). -func Parse(output []byte) (desc, text string, start int, end int) { +func Parse(output []byte) (desc string, text []byte, start int, end int) { var oops *oops - var textData []byte for pos := 0; pos < len(output); { next := bytes.IndexByte(output[pos:], '\n') if next != -1 { @@ -216,8 +215,8 @@ func Parse(output []byte) (desc, text string, start int, end int) { if lineEnd != 0 && output[lineEnd-1] == '\r' { lineEnd-- } - textData = append(textData, output[lineStart:lineEnd]...) - textData = append(textData, '\n') + text = append(text, output[lineStart:lineEnd]...) + text = append(text, '\n') } } pos = next + 1 @@ -225,7 +224,6 @@ func Parse(output []byte) (desc, text string, start int, end int) { if oops == nil { return } - text = string(textData) desc = extractDescription(output[start:], oops) if len(desc) > 0 && desc[len(desc)-1] == '\r' { desc = desc[:len(desc)-1] @@ -258,11 +256,11 @@ func extractDescription(output []byte, oops *oops) string { return string(output[pos:end]) } -func Symbolize(vmlinux, text string) (string, error) { +func Symbolize(vmlinux string, text []byte) ([]byte, error) { var symbolized []byte symbols, err := symbolizer.ReadSymbols(vmlinux) if err != nil { - return "", err + return nil, err } symb := symbolizer.NewSymbolizer() symbFunc := func(bin string, pc uint64) ([]symbolizer.Frame, error) { @@ -270,14 +268,14 @@ func Symbolize(vmlinux, text string) (string, error) { } strip, _ := filepath.Abs(vmlinux) strip = filepath.Dir(strip) + string(filepath.Separator) - s := bufio.NewScanner(strings.NewReader(text)) + s := bufio.NewScanner(bytes.NewReader(text)) for s.Scan() { line := append([]byte{}, s.Bytes()...) line = append(line, '\n') line = symbolizeLine(symbFunc, symbols, vmlinux, strip, line) symbolized = append(symbolized, line...) } - return string(symbolized), nil + return symbolized, nil } func symbolizeLine(symbFunc func(bin string, pc uint64) ([]symbolizer.Frame, error), symbols map[string][]symbolizer.Symbol, vmlinux, strip string, line []byte) []byte { diff --git a/syz-manager/manager.go b/syz-manager/manager.go index 85422fe4..c0e1d2dd 100644 --- a/syz-manager/manager.go +++ b/syz-manager/manager.go @@ -84,7 +84,6 @@ func main() { func RunManager(cfg *config.Config, syscalls map[int]bool, suppressions []*regexp.Regexp) { crashdir := filepath.Join(cfg.Workdir, "crashes") - os.MkdirAll(crashdir, 0700) enabledSyscalls := "" if len(syscalls) != 0 { @@ -259,47 +258,8 @@ func (mgr *Manager) runInstance(vmCfg *vm.Config, first bool) bool { logf(0, "failed to run fuzzer: %v", err) return false } - startTime := time.Now() - var crashes []string - - saveCrasher := func(what string, output []byte) { - if atomic.LoadUint32(&mgr.shutdown) != 0 { - // qemu crashes with "qemu: terminating on signal 2", - // which we detect as "lost connection". - return - } - for _, re := range mgr.suppressions { - if re.Match(output) { - logf(1, "%v: suppressing '%v' with '%v'", vmCfg.Name, what, re.String()) - mgr.mu.Lock() - mgr.stats["suppressed"]++ - mgr.mu.Unlock() - return - } - } - buf := new(bytes.Buffer) - fmt.Fprintf(buf, "\n\n") - if len(crashes) != 0 { - fmt.Fprintf(buf, "previous crashes:\n") - for _, c := range crashes { - fmt.Fprintf(buf, "\t%s\n", c) - } - } - crashes = append(crashes, what) - fmt.Fprintf(buf, "after running for %v:\n", time.Since(startTime)) - fmt.Fprintf(buf, "%v\n", what) - output = append([]byte{}, output...) - output = append(output, buf.Bytes()...) - filename := fmt.Sprintf("crash-%v-%v", vmCfg.Name, time.Now().UnixNano()) - logf(0, "%v: saving crash '%v' to %v", vmCfg.Name, what, filename) - ioutil.WriteFile(filepath.Join(mgr.crashdir, filename), output, 0660) - mgr.mu.Lock() - mgr.stats["crashes"]++ - mgr.mu.Unlock() - } var output []byte - waitForOutput := func(dur time.Duration) { timer := time.NewTimer(dur).C for { @@ -344,7 +304,7 @@ func (mgr *Manager) runInstance(vmCfg *vm.Config, first bool) bool { return true default: waitForOutput(10 * time.Second) - saveCrasher("lost connection", output) + mgr.saveCrasher(vmCfg, "lost connection to test machine", nil, output) return true } case out := <-outputC: @@ -355,7 +315,7 @@ func (mgr *Manager) runInstance(vmCfg *vm.Config, first bool) bool { if report.ContainsCrash(output[matchPos:]) { // Give it some time to finish writing the error message. waitForOutput(10 * time.Second) - desc, _, start, end := report.Parse(output[matchPos:]) + desc, text, start, end := report.Parse(output[matchPos:]) start = start + matchPos - beforeContext if start < 0 { start = 0 @@ -364,7 +324,7 @@ func (mgr *Manager) runInstance(vmCfg *vm.Config, first bool) bool { if end > len(output) { end = len(output) } - saveCrasher(desc, output[start:end]) + mgr.saveCrasher(vmCfg, desc, text, output[start:end]) return true } if len(output) > 2*beforeContext { @@ -379,19 +339,70 @@ func (mgr *Manager) runInstance(vmCfg *vm.Config, first bool) bool { // but fuzzer is not actually executing programs. if mgr.cfg.Type != "local" && time.Since(lastExecuteTime) > 3*time.Minute { dumpVMState() - saveCrasher("not executing programs", output) + mgr.saveCrasher(vmCfg, "test machine is not executing programs", nil, output) return true } case <-ticker.C: if mgr.cfg.Type != "local" { dumpVMState() - saveCrasher("no output", output) + mgr.saveCrasher(vmCfg, "no output from test machine", nil, output) return true } } } } +func (mgr *Manager) saveCrasher(vmCfg *vm.Config, desc string, text, output []byte) { + if atomic.LoadUint32(&mgr.shutdown) != 0 { + // qemu crashes with "qemu: terminating on signal 2", + // which we detect as "lost connection". + return + } + for _, re := range mgr.suppressions { + if re.Match(output) { + logf(1, "%v: suppressing '%v' with '%v'", vmCfg.Name, desc, re.String()) + mgr.mu.Lock() + mgr.stats["suppressed"]++ + mgr.mu.Unlock() + return + } + } + + logf(0, "%v: crash: %v", vmCfg.Name, desc) + mgr.mu.Lock() + mgr.stats["crashes"]++ + mgr.mu.Unlock() + + h := hash([]byte(desc)) + id := hex.EncodeToString(h[:]) + dir := filepath.Join(mgr.crashdir, id) + os.MkdirAll(dir, 0700) + if err := ioutil.WriteFile(filepath.Join(dir, "description"), []byte(desc+"\n"), 0660); err != nil { + logf(0, "failed to write crash: %v", err) + } + const maxReports = 100 // save up to 100 reports + if matches, _ := filepath.Glob(filepath.Join(dir, "log*")); len(matches) >= maxReports { + return + } + for i := 0; i < maxReports; i++ { + fn := filepath.Join(dir, fmt.Sprintf("log%v", i)) + if _, err := os.Stat(fn); err == nil { + continue + } + if err := ioutil.WriteFile(fn, output, 0660); err != nil { + continue + } + symbolized, err := report.Symbolize(mgr.cfg.Vmlinux, text) + if err != nil { + logf(0, "failed to symbolize crash: %v", err) + } else { + text = symbolized + } + ioutil.WriteFile(filepath.Join(dir, fmt.Sprintf("report%v", i)), []byte(text), 0660) + break + } +} + func (mgr *Manager) minimizeCorpus() { if mgr.cfg.Cover && len(mgr.corpus) != 0 { // First, sort corpus per call.