diff --git a/Makefile b/Makefile index b374b3c5..ce54678f 100644 --- a/Makefile +++ b/Makefile @@ -75,7 +75,7 @@ endif .PHONY: all host target \ manager fuzzer executor \ ci hub \ - execprog mutate prog2c stress repro upgrade db parse \ + execprog mutate prog2c stress repro upgrade db \ bin/syz-sysgen bin/syz-extract bin/syz-fmt \ extract generate generate_go generate_sys \ format format_go format_cpp format_sys \ @@ -91,7 +91,7 @@ all: host target host: GOOS=$(HOSTOS) GOARCH=$(HOSTARCH) $(GO) install ./syz-manager - $(MAKE) manager repro mutate prog2c db parse upgrade + $(MAKE) manager repro mutate prog2c db upgrade target: GOOS=$(TARGETOS) GOARCH=$(TARGETVMARCH) $(GO) install ./syz-fuzzer @@ -134,9 +134,6 @@ stress: db: GOOS=$(HOSTOS) GOARCH=$(HOSTARCH) $(GO) build $(GOFLAGS) -o ./bin/syz-db github.com/google/syzkaller/tools/syz-db -parse: - GOOS=$(HOSTOS) GOARCH=$(HOSTARCH) $(GO) build $(GOFLAGS) -o ./bin/syz-parse github.com/google/syzkaller/tools/syz-parse - upgrade: GOOS=$(HOSTOS) GOARCH=$(HOSTARCH) $(GO) build $(GOFLAGS) -o ./bin/syz-upgrade github.com/google/syzkaller/tools/syz-upgrade diff --git a/pkg/instance/instance.go b/pkg/instance/instance.go index b5f44376..e7ab3c9e 100644 --- a/pkg/instance/instance.go +++ b/pkg/instance/instance.go @@ -82,7 +82,7 @@ func (env *Env) BuildKernel(compilerBin, userspaceDir, cmdlineFile, sysctlFile s if err := kernel.Build(cfg.KernelSrc, compilerBin, kernelConfig); err != nil { return osutil.PrependContext("kernel build failed", err) } - cfg.Vmlinux = filepath.Join(cfg.KernelSrc, "vmlinux") + cfg.KernelObj = cfg.KernelSrc cfg.Image = filepath.Join(cfg.Workdir, "syz-image") cfg.SSHKey = filepath.Join(cfg.Workdir, "syz-key") if err := kernel.CreateImage(cfg.TargetOS, cfg.TargetVMArch, cfg.Type, @@ -118,13 +118,11 @@ func (env *Env) Test(numVMs int, reproSyz, reproOpts, reproC []byte) ([]error, e if err := mgrconfig.Complete(env.cfg); err != nil { return nil, err } - reporter, err := report.NewReporter(env.cfg.TargetOS, env.cfg.Type, - env.cfg.KernelSrc, filepath.Dir(env.cfg.Vmlinux), nil, env.cfg.ParsedIgnores) + reporter, err := report.NewReporter(env.cfg) if err != nil { return nil, err } - vmEnv := mgrconfig.CreateVMEnv(env.cfg, false) - vmPool, err := vm.Create(env.cfg.Type, vmEnv) + vmPool, err := vm.Create(env.cfg, false) if err != nil { return nil, fmt.Errorf("failed to create VM pool: %v", err) } diff --git a/pkg/report/freebsd.go b/pkg/report/freebsd.go index f16f2c15..6e3bbb7e 100644 --- a/pkg/report/freebsd.go +++ b/pkg/report/freebsd.go @@ -6,26 +6,21 @@ package report import ( "bytes" "regexp" - - "github.com/google/syzkaller/pkg/symbolizer" ) type freebsd struct { kernelSrc string kernelObj string - symbols map[string][]symbolizer.Symbol ignores []*regexp.Regexp } -func ctorFreebsd(kernelSrc, kernelObj string, symbols map[string][]symbolizer.Symbol, - ignores []*regexp.Regexp) (Reporter, error) { +func ctorFreebsd(kernelSrc, kernelObj string, ignores []*regexp.Regexp) (Reporter, []string, error) { ctx := &freebsd{ kernelSrc: kernelSrc, kernelObj: kernelObj, - symbols: symbols, ignores: ignores, } - return ctx, nil + return ctx, nil, nil } func (ctx *freebsd) ContainsCrash(output []byte) bool { diff --git a/pkg/report/fuchsia.go b/pkg/report/fuchsia.go index 67a3e2ae..4e227fc9 100644 --- a/pkg/report/fuchsia.go +++ b/pkg/report/fuchsia.go @@ -6,26 +6,21 @@ package report import ( "bytes" "regexp" - - "github.com/google/syzkaller/pkg/symbolizer" ) type fuchsia struct { kernelSrc string kernelObj string - symbols map[string][]symbolizer.Symbol ignores []*regexp.Regexp } -func ctorFuchsia(kernelSrc, kernelObj string, symbols map[string][]symbolizer.Symbol, - ignores []*regexp.Regexp) (Reporter, error) { +func ctorFuchsia(kernelSrc, kernelObj string, ignores []*regexp.Regexp) (Reporter, []string, error) { ctx := &fuchsia{ kernelSrc: kernelSrc, kernelObj: kernelObj, - symbols: symbols, ignores: ignores, } - return ctx, nil + return ctx, nil, nil } func (ctx *fuchsia) ContainsCrash(output []byte) bool { diff --git a/pkg/report/gvisor.go b/pkg/report/gvisor.go index 357051fb..fa1c8476 100644 --- a/pkg/report/gvisor.go +++ b/pkg/report/gvisor.go @@ -6,20 +6,23 @@ package report import ( "bytes" "regexp" - - "github.com/google/syzkaller/pkg/symbolizer" ) type gvisor struct { ignores []*regexp.Regexp } -func ctorGvisor(kernelSrc, kernelObj string, symbols map[string][]symbolizer.Symbol, - ignores []*regexp.Regexp) (Reporter, error) { +func ctorGvisor(kernelSrc, kernelObj string, ignores []*regexp.Regexp) (Reporter, []string, error) { ctx := &gvisor{ ignores: ignores, } - return ctx, nil + suppressions := []string{ + "fatal error: runtime: out of memory", + "fatal error: runtime: cannot allocate memory", + "panic: failed to start executor binary", + "panic: executor failed: pthread_create failed", + } + return ctx, suppressions, nil } func (ctx *gvisor) ContainsCrash(output []byte) bool { diff --git a/pkg/report/linux.go b/pkg/report/linux.go index 0fc67f80..f29235b6 100644 --- a/pkg/report/linux.go +++ b/pkg/report/linux.go @@ -32,17 +32,15 @@ type linux struct { eoi []byte } -func ctorLinux(kernelSrc, kernelObj string, symbols map[string][]symbolizer.Symbol, - ignores []*regexp.Regexp) (Reporter, error) { +func ctorLinux(kernelSrc, kernelObj string, ignores []*regexp.Regexp) (Reporter, []string, error) { vmlinux := "" + var symbols map[string][]symbolizer.Symbol if kernelObj != "" { vmlinux = filepath.Join(kernelObj, "vmlinux") - if symbols == nil { - var err error - symbols, err = symbolizer.ReadSymbols(vmlinux) - if err != nil { - return nil, err - } + var err error + symbols, err = symbolizer.ReadSymbols(vmlinux) + if err != nil { + return nil, nil, err } } ctx := &linux{ @@ -95,7 +93,23 @@ func ctorLinux(kernelSrc, kernelObj string, symbols map[string][]symbolizer.Symb []byte("FAULT_INJECTION: forcing a failure"), []byte("FAULT_FLAG_ALLOW_RETRY missing"), } - return ctx, nil + suppressions := []string{ + "fatal error: runtime: out of memory", + "fatal error: runtime: cannot allocate memory", + "panic: failed to start executor binary", + "panic: executor failed: pthread_create failed", + "panic: failed to create temp dir", + "fatal error: unexpected signal during runtime execution", // presubmably OOM turned into SIGBUS + "signal SIGBUS: bus error", // presubmably OOM turned into SIGBUS + "Out of memory: Kill process .* \\(syz-fuzzer\\)", + "Out of memory: Kill process .* \\(sshd\\)", + "Killed process .* \\(syz-fuzzer\\)", + "Killed process .* \\(sshd\\)", + "lowmemorykiller: Killing 'syz-fuzzer'", + "lowmemorykiller: Killing 'sshd'", + "INIT: PANIC: segmentation violation!", + } + return ctx, suppressions, nil } func (ctx *linux) ContainsCrash(output []byte) bool { diff --git a/pkg/report/linux_test.go b/pkg/report/linux_test.go index 0543fcbe..338ab27a 100644 --- a/pkg/report/linux_test.go +++ b/pkg/report/linux_test.go @@ -5,38 +5,32 @@ package report import ( "fmt" - "regexp" "testing" "github.com/google/syzkaller/pkg/symbolizer" + "github.com/google/syzkaller/syz-manager/mgrconfig" ) func TestLinuxIgnores(t *testing.T) { - reporter, err := NewReporter("linux", "", "", "", nil, nil) + cfg := &mgrconfig.Config{ + TargetOS: "linux", + } + reporter, err := NewReporter(cfg) if err != nil { t.Fatal(err) } - ignores1 := []*regexp.Regexp{ - regexp.MustCompile("BUG: bug3"), - } - reporter1, err := NewReporter("linux", "", "", "", nil, ignores1) + cfg.Ignores = []string{"BUG: bug3"} + reporter1, err := NewReporter(cfg) if err != nil { t.Fatal(err) } - ignores2 := []*regexp.Regexp{ - regexp.MustCompile("BUG: bug3"), - regexp.MustCompile("BUG: bug1"), - } - reporter2, err := NewReporter("linux", "", "", "", nil, ignores2) + cfg.Ignores = []string{"BUG: bug3", "BUG: bug1"} + reporter2, err := NewReporter(cfg) if err != nil { t.Fatal(err) } - ignores3 := []*regexp.Regexp{ - regexp.MustCompile("BUG: bug3"), - regexp.MustCompile("BUG: bug1"), - regexp.MustCompile("BUG: bug2"), - } - reporter3, err := NewReporter("linux", "", "", "", nil, ignores3) + cfg.Ignores = []string{"BUG: bug3", "BUG: bug1", "BUG: bug2"} + reporter3, err := NewReporter(cfg) if err != nil { t.Fatal(err) } diff --git a/pkg/report/netbsd.go b/pkg/report/netbsd.go index 6c4dbedf..5457327c 100644 --- a/pkg/report/netbsd.go +++ b/pkg/report/netbsd.go @@ -5,26 +5,21 @@ package report import ( "regexp" - - "github.com/google/syzkaller/pkg/symbolizer" ) type netbsd struct { kernelSrc string kernelObj string - symbols map[string][]symbolizer.Symbol ignores []*regexp.Regexp } -func ctorNetbsd(kernelSrc, kernelObj string, symbols map[string][]symbolizer.Symbol, - ignores []*regexp.Regexp) (Reporter, error) { +func ctorNetbsd(kernelSrc, kernelObj string, ignores []*regexp.Regexp) (Reporter, []string, error) { ctx := &netbsd{ kernelSrc: kernelSrc, kernelObj: kernelObj, - symbols: symbols, ignores: ignores, } - return ctx, nil + return ctx, nil, nil } func (ctx *netbsd) ContainsCrash(output []byte) bool { diff --git a/pkg/report/report.go b/pkg/report/report.go index bd702b1d..2e8d13a0 100644 --- a/pkg/report/report.go +++ b/pkg/report/report.go @@ -12,7 +12,7 @@ import ( "regexp" "strings" - "github.com/google/syzkaller/pkg/symbolizer" + "github.com/google/syzkaller/syz-manager/mgrconfig" ) type Reporter interface { @@ -37,6 +37,8 @@ type Report struct { // StartPos/EndPos denote region of output with oops message(s). StartPos int EndPos int + // Suppressed indicates whether the report should not be reported to user. + Suppressed bool // Corrupted indicates whether the report is truncated of corrupted in some other way. Corrupted bool // corruptedReason contains reason why the report is marked as corrupted. @@ -45,28 +47,29 @@ type Report struct { Maintainers []string } -// NewReporter creates reporter for the specified OS/vmType: -// kernelSrc: path to kernel sources directory -// kernelObj: path to kernel build directory (can be empty for in-tree build) -// symbols: kernel symbols (result of pkg/symbolizer.ReadSymbols on kernel object file) -// ignores: optional list of regexps to ignore (must match first line of crash message) -func NewReporter(os, vmType, kernelSrc, kernelObj string, symbols map[string][]symbolizer.Symbol, - ignores []*regexp.Regexp) (Reporter, error) { - if vmType == "gvisor" { - os = vmType +// NewReporter creates reporter for the specified OS/Type. +func NewReporter(cfg *mgrconfig.Config) (Reporter, error) { + typ := cfg.TargetOS + if cfg.Type == "gvisor" { + typ = cfg.Type } - ctor := ctors[os] + ctor := ctors[typ] if ctor == nil { - return nil, fmt.Errorf("unknown os: %v", os) + return nil, fmt.Errorf("unknown OS: %v", typ) } - if kernelObj == "" { - kernelObj = kernelSrc // assume in-tree build - } - rep, err := ctor(kernelSrc, kernelObj, symbols, ignores) + ignores, err := compileRegexps(cfg.Ignores) if err != nil { return nil, err } - return reporterWrapper{rep}, nil + rep, suppressions, err := ctor(cfg.KernelSrc, cfg.KernelObj, ignores) + if err != nil { + return nil, err + } + supps, err := compileRegexps(append(suppressions, cfg.Suppressions...)) + if err != nil { + return nil, err + } + return reporterWrapper{rep, supps}, nil } var ctors = map[string]fn{ @@ -79,10 +82,23 @@ var ctors = map[string]fn{ "windows": ctorStub, } -type fn func(string, string, map[string][]symbolizer.Symbol, []*regexp.Regexp) (Reporter, error) +type fn func(string, string, []*regexp.Regexp) (Reporter, []string, error) + +func compileRegexps(list []string) ([]*regexp.Regexp, error) { + compiled := make([]*regexp.Regexp, len(list)) + for i, str := range list { + re, err := regexp.Compile(str) + if err != nil { + return nil, fmt.Errorf("failed to compile %q: %v", str, err) + } + compiled[i] = re + } + return compiled, nil +} type reporterWrapper struct { Reporter + suppressions []*regexp.Regexp } func (wrap reporterWrapper) Parse(output []byte) *Report { @@ -91,6 +107,7 @@ func (wrap reporterWrapper) Parse(output []byte) *Report { return nil } rep.Title = sanitizeTitle(replaceTable(dynamicTitleReplacement, rep.Title)) + rep.Suppressed = matchesAny(rep.Output, wrap.suppressions) return rep } diff --git a/pkg/report/report_test.go b/pkg/report/report_test.go index 14d6c45c..a2d83383 100644 --- a/pkg/report/report_test.go +++ b/pkg/report/report_test.go @@ -16,6 +16,7 @@ import ( "testing" "github.com/google/syzkaller/pkg/osutil" + "github.com/google/syzkaller/syz-manager/mgrconfig" ) var flagUpdate = flag.Bool("update", false, "update test files accordingly to current results") @@ -214,7 +215,10 @@ func forEachFile(t *testing.T, dir string, fn func(t *testing.T, reporter Report if err != nil { t.Fatal(err) } - reporter, err := NewReporter(os, "", "", "", nil, nil) + cfg := &mgrconfig.Config{ + TargetOS: os, + } + reporter, err := NewReporter(cfg) if err != nil { t.Fatal(err) } diff --git a/pkg/report/stub.go b/pkg/report/stub.go index 04f17084..32bcb008 100644 --- a/pkg/report/stub.go +++ b/pkg/report/stub.go @@ -5,26 +5,21 @@ package report import ( "regexp" - - "github.com/google/syzkaller/pkg/symbolizer" ) type stub struct { kernelSrc string kernelObj string - symbols map[string][]symbolizer.Symbol ignores []*regexp.Regexp } -func ctorStub(kernelSrc, kernelObj string, symbols map[string][]symbolizer.Symbol, - ignores []*regexp.Regexp) (Reporter, error) { +func ctorStub(kernelSrc, kernelObj string, ignores []*regexp.Regexp) (Reporter, []string, error) { ctx := &stub{ kernelSrc: kernelSrc, kernelObj: kernelObj, - symbols: symbols, ignores: ignores, } - return ctx, nil + return ctx, nil, nil } func (ctx *stub) ContainsCrash(output []byte) bool { diff --git a/syz-ci/jobs.go b/syz-ci/jobs.go index d4b1e51b..3866c8a6 100644 --- a/syz-ci/jobs.go +++ b/syz-ci/jobs.go @@ -168,7 +168,7 @@ func (jp *JobProcessor) test(job *Job) error { mgrcfg.Name += "-job" mgrcfg.Workdir = filepath.Join(dir, "workdir") mgrcfg.KernelSrc = kernelDir - mgrcfg.Vmlinux = filepath.Join(kernelDir, "vmlinux") + mgrcfg.KernelObj = kernelDir mgrcfg.Syzkaller = filepath.Join(dir, "gopath", "src", "github.com", "google", "syzkaller") os.RemoveAll(mgrcfg.Workdir) diff --git a/syz-ci/manager.go b/syz-ci/manager.go index 92ad715e..1341004c 100644 --- a/syz-ci/manager.go +++ b/syz-ci/manager.go @@ -435,9 +435,9 @@ func (mgr *Manager) createTestConfig(imageDir string, info *BuildInfo) (*mgrconf mgrcfg.Name += "-test" mgrcfg.Tag = info.KernelCommit mgrcfg.Workdir = filepath.Join(imageDir, "workdir") - mgrcfg.Vmlinux = filepath.Join(imageDir, "obj", "vmlinux") mgrcfg.Image = filepath.Join(imageDir, "image") mgrcfg.SSHKey = filepath.Join(imageDir, "key") + mgrcfg.KernelObj = filepath.Join(imageDir, "obj") mgrcfg.KernelSrc = mgr.kernelDir if err := mgrconfig.Complete(mgrcfg); err != nil { return nil, fmt.Errorf("bad manager config: %v", err) @@ -461,7 +461,7 @@ func (mgr *Manager) writeConfig(buildTag string) (string, error) { } mgrcfg.Tag = buildTag mgrcfg.Workdir = mgr.workDir - mgrcfg.Vmlinux = filepath.Join(mgr.currentDir, "obj", "vmlinux") + mgrcfg.KernelObj = filepath.Join(mgr.currentDir, "obj") // Strictly saying this is somewhat racy as builder can concurrently // update the source, or even delete and re-clone. If this causes // problems, we need to make a copy of sources after build. diff --git a/syz-manager/cover.go b/syz-manager/cover.go index d78d1c20..953fa3b8 100644 --- a/syz-manager/cover.go +++ b/syz-manager/cover.go @@ -14,11 +14,11 @@ import ( "sort" "strconv" "strings" + "sync" "time" "github.com/google/syzkaller/pkg/cover" "github.com/google/syzkaller/pkg/hash" - "github.com/google/syzkaller/pkg/log" "github.com/google/syzkaller/pkg/osutil" "github.com/google/syzkaller/pkg/symbolizer" ) @@ -29,79 +29,63 @@ type symbol struct { name string } -type symbolArray []symbol - -func (a symbolArray) Len() int { return len(a) } -func (a symbolArray) Less(i, j int) bool { return a[i].start < a[j].start } -func (a symbolArray) Swap(i, j int) { a[i], a[j] = a[j], a[i] } - type coverage struct { line int covered bool } -type coverageArray []coverage - -func (a coverageArray) Len() int { return len(a) } -func (a coverageArray) Less(i, j int) bool { return a[i].line < a[j].line } -func (a coverageArray) Swap(i, j int) { a[i], a[j] = a[j], a[i] } - -type uint64Array []uint64 - -func (a uint64Array) Len() int { return len(a) } -func (a uint64Array) Less(i, j int) bool { return a[i] < a[j] } -func (a uint64Array) Swap(i, j int) { a[i], a[j] = a[j], a[i] } - var ( - allCoverPCs []uint64 - allCoverReady = make(chan bool) - allSymbols map[string][]symbolizer.Symbol - allSymbolsReady = make(chan bool) - vmOffsets = make(map[string]uint32) + initCoverOnce sync.Once + initCoverError error + initCoverSymbols []symbol + initCoverPCs []uint64 + initCoverVMOffset uint32 ) -func initAllCover(os, arch, vmlinux string) { - // Running objdump on vmlinux takes 20-30 seconds, so we do it asynchronously on start. - // Running nm on vmlinux may takes 200 microsecond and being called during symbolization of every crash, - // so also do it asynchronously on start and reuse the value during each crash. - go func() { - defer func() { - close(allCoverReady) - close(allSymbolsReady) - }() - if vmlinux == "" { - return +func initCover(kernelObj, arch string) error { + if kernelObj == "" { + return fmt.Errorf("kernel_obj is not specified") + } + vmlinux := filepath.Join(kernelObj, "vmlinux") + symbols, err := symbolizer.ReadSymbols(vmlinux) + if err != nil { + return fmt.Errorf("failed to run nm on %v: %v", vmlinux, err) + } + for name, ss := range symbols { + for _, s := range ss { + initCoverSymbols = append(initCoverSymbols, symbol{s.Addr, s.Addr + uint64(s.Size), name}) } - pcs, err := coveredPCs(arch, vmlinux) - if err == nil { - sort.Sort(uint64Array(pcs)) - allCoverPCs = pcs - } else { - log.Logf(0, "failed to run objdump on %v: %v", vmlinux, err) - } - - allSymbols, err = symbolizer.ReadSymbols(vmlinux) - if err != nil { - log.Logf(0, "failed to run nm on %v: %v", vmlinux, err) - } - }() + } + sort.Slice(initCoverSymbols, func(i, j int) bool { + return initCoverSymbols[i].start < initCoverSymbols[j].start + }) + initCoverPCs, err = coveredPCs(arch, vmlinux) + if err != nil { + return fmt.Errorf("failed to run objdump on %v: %v", vmlinux, err) + } + sort.Slice(initCoverPCs, func(i, j int) bool { + return initCoverPCs[i] < initCoverPCs[j] + }) + initCoverVMOffset, err = getVMOffset(vmlinux) + return err } -func generateCoverHTML(w io.Writer, vmlinux, arch string, cov cover.Cover) error { +func generateCoverHTML(w io.Writer, kernelObj, arch string, cov cover.Cover) error { if len(cov) == 0 { - return fmt.Errorf("No coverage data available") + return fmt.Errorf("no coverage data available") + } + initCoverOnce.Do(func() { initCoverError = initCover(kernelObj, arch) }) + if initCoverError != nil { + return initCoverError } - base, err := getVMOffset(vmlinux) - if err != nil { - return err - } pcs := make([]uint64, 0, len(cov)) for pc := range cov { - fullPC := cover.RestorePC(pc, base) + fullPC := cover.RestorePC(pc, initCoverVMOffset) prevPC := previousInstructionPC(arch, fullPC) pcs = append(pcs, prevPC) } + vmlinux := filepath.Join(kernelObj, "vmlinux") uncovered, err := uncoveredPcsInFuncs(vmlinux, pcs) if err != nil { return err @@ -186,7 +170,9 @@ func fileSet(covered, uncovered []symbolizer.Frame) map[string][]coverage { for ln, covered := range lines { sorted = append(sorted, coverage{ln, covered}) } - sort.Sort(coverageArray(sorted)) + sort.Slice(sorted, func(i, j int) bool { + return sorted[i].line < sorted[j].line + }) res[f] = sorted } return res @@ -214,9 +200,6 @@ func parseFile(fn string) ([][]byte, error) { } func getVMOffset(vmlinux string) (uint32, error) { - if v, ok := vmOffsets[vmlinux]; ok { - return v, nil - } out, err := osutil.RunCmd(time.Hour, "", "readelf", "-SW", vmlinux) if err != nil { return 0, err @@ -246,51 +229,33 @@ func getVMOffset(vmlinux string) (uint32, error) { } } } - vmOffsets[vmlinux] = addr return addr, nil } // uncoveredPcsInFuncs returns uncovered PCs with __sanitizer_cov_trace_pc calls in functions containing pcs. func uncoveredPcsInFuncs(vmlinux string, pcs []uint64) ([]uint64, error) { - <-allSymbolsReady - if allSymbols == nil { - return nil, fmt.Errorf("failed to run nm on vmlinux") - } - var symbols symbolArray - for name, ss := range allSymbols { - for _, s := range ss { - symbols = append(symbols, symbol{s.Addr, s.Addr + uint64(s.Size), name}) - } - } - sort.Sort(symbols) - - <-allCoverReady - if len(allCoverPCs) == 0 { - return nil, nil - } - handledFuncs := make(map[uint64]bool) uncovered := make(map[uint64]bool) for _, pc := range pcs { - idx := sort.Search(len(symbols), func(i int) bool { - return pc < symbols[i].end + idx := sort.Search(len(initCoverSymbols), func(i int) bool { + return pc < initCoverSymbols[i].end }) - if idx == len(symbols) { + if idx == len(initCoverSymbols) { continue } - s := symbols[idx] + s := initCoverSymbols[idx] if pc < s.start || pc > s.end { continue } if !handledFuncs[s.start] { handledFuncs[s.start] = true - startPC := sort.Search(len(allCoverPCs), func(i int) bool { - return s.start <= allCoverPCs[i] + startPC := sort.Search(len(initCoverPCs), func(i int) bool { + return s.start <= initCoverPCs[i] }) - endPC := sort.Search(len(allCoverPCs), func(i int) bool { - return s.end < allCoverPCs[i] + endPC := sort.Search(len(initCoverPCs), func(i int) bool { + return s.end < initCoverPCs[i] }) - for _, pc1 := range allCoverPCs[startPC:endPC] { + for _, pc1 := range initCoverPCs[startPC:endPC] { uncovered[pc1] = true } } diff --git a/syz-manager/html.go b/syz-manager/html.go index d8997c0d..17a61c4c 100644 --- a/syz-manager/html.go +++ b/syz-manager/html.go @@ -202,8 +202,8 @@ func (mgr *Manager) httpCover(w http.ResponseWriter, r *http.Request) { mgr.mu.Lock() defer mgr.mu.Unlock() - if mgr.cfg.Vmlinux == "" { - http.Error(w, fmt.Sprintf("no vmlinux in config file"), http.StatusInternalServerError) + if mgr.cfg.KernelObj == "" { + http.Error(w, fmt.Sprintf("no kernel_obj in config file"), http.StatusInternalServerError) return } var cov cover.Cover @@ -218,7 +218,7 @@ func (mgr *Manager) httpCover(w http.ResponseWriter, r *http.Request) { } } - if err := generateCoverHTML(w, mgr.cfg.Vmlinux, mgr.cfg.TargetVMArch, cov); err != nil { + if err := generateCoverHTML(w, mgr.cfg.KernelObj, mgr.cfg.TargetVMArch, cov); err != nil { http.Error(w, fmt.Sprintf("failed to generate coverage profile: %v", err), http.StatusInternalServerError) return } @@ -317,9 +317,9 @@ func (mgr *Manager) httpRawCover(w http.ResponseWriter, r *http.Request) { mgr.mu.Lock() defer mgr.mu.Unlock() - base, err := getVMOffset(mgr.cfg.Vmlinux) - if err != nil { - http.Error(w, fmt.Sprintf("failed to get vmlinux base: %v", err), http.StatusInternalServerError) + initCoverOnce.Do(func() { initCoverError = initCover(mgr.cfg.KernelObj, mgr.cfg.TargetArch) }) + if initCoverError != nil { + http.Error(w, initCoverError.Error(), http.StatusInternalServerError) return } @@ -329,7 +329,7 @@ func (mgr *Manager) httpRawCover(w http.ResponseWriter, r *http.Request) { } pcs := make([]uint64, 0, len(cov)) for pc := range cov { - fullPC := cover.RestorePC(pc, base) + fullPC := cover.RestorePC(pc, initCoverVMOffset) prevPC := previousInstructionPC(mgr.cfg.TargetVMArch, fullPC) pcs = append(pcs, prevPC) } diff --git a/syz-manager/manager.go b/syz-manager/manager.go index c99cd5e1..5d6d2ffc 100644 --- a/syz-manager/manager.go +++ b/syz-manager/manager.go @@ -45,7 +45,6 @@ type Manager struct { cfg *mgrconfig.Config vmPool *vm.Pool target *prog.Target - reporterInit sync.Once reporter report.Reporter crashdir string port int @@ -136,7 +135,6 @@ func main() { if err != nil { log.Fatalf("%v", err) } - initAllCover(cfg.TargetOS, cfg.TargetVMArch, cfg.Vmlinux) RunManager(cfg, target, syscalls) } @@ -146,9 +144,8 @@ func RunManager(cfg *mgrconfig.Config, target *prog.Target, syscalls map[int]boo // does not start any VMs, but instead you start them manually // and start syz-fuzzer there. if cfg.Type != "none" { - env := mgrconfig.CreateVMEnv(cfg, *flagDebug) var err error - vmPool, err = vm.Create(cfg.Type, env) + vmPool, err = vm.Create(cfg, *flagDebug) if err != nil { log.Fatalf("%v", err) } @@ -162,10 +159,16 @@ func RunManager(cfg *mgrconfig.Config, target *prog.Target, syscalls map[int]boo enabledSyscalls = append(enabledSyscalls, c) } + reporter, err := report.NewReporter(cfg) + if err != nil { + log.Fatalf("%v", err) + } + mgr := &Manager{ cfg: cfg, vmPool: vmPool, target: target, + reporter: reporter, crashdir: crashdir, startTime: time.Now(), stats: make(map[string]uint64), @@ -183,7 +186,6 @@ func RunManager(cfg *mgrconfig.Config, target *prog.Target, syscalls map[int]boo } log.Logf(0, "loading corpus...") - var err error mgr.corpusDB, err = db.Open(filepath.Join(cfg.Workdir, "corpus.db")) if err != nil { log.Fatalf("failed to open corpus database: %v", err) @@ -384,7 +386,7 @@ func (mgr *Manager) vmLoop() { atomic.AddUint32(&mgr.numReproducing, 1) log.Logf(1, "loop: starting repro of '%v' on instances %+v", crash.Title, vmIndexes) go func() { - res, err := repro.Run(crash.Output, mgr.cfg, mgr.getReporter(), mgr.vmPool, vmIndexes) + res, err := repro.Run(crash.Output, mgr.cfg, mgr.reporter, mgr.vmPool, vmIndexes) reproDone <- &ReproResult{vmIndexes, crash.Title, res, err, crash.hub} }() } @@ -418,7 +420,7 @@ func (mgr *Manager) vmLoop() { instances = append(instances, res.idx) // On shutdown qemu crashes with "qemu: terminating on signal 2", // which we detect as "lost connection". Don't save that as crash. - if shutdown != nil && res.crash != nil && !mgr.isSuppressed(res.crash) { + if shutdown != nil && res.crash != nil { needRepro := mgr.saveCrash(res.crash) if needRepro { log.Logf(1, "loop: add pending repro for '%v'", res.crash.Title) @@ -584,7 +586,7 @@ func (mgr *Manager) runInstance(index int) (*Crash, error) { return nil, fmt.Errorf("failed to run fuzzer: %v", err) } - rep := inst.MonitorExecution(outc, errc, mgr.getReporter(), false) + rep := inst.MonitorExecution(outc, errc, mgr.reporter, false) if rep == nil { // This is the only "OK" outcome. log.Logf(0, "vm-%v: running for %v, restarting", index, time.Since(start)) @@ -598,20 +600,6 @@ func (mgr *Manager) runInstance(index int) (*Crash, error) { return cash, nil } -func (mgr *Manager) isSuppressed(crash *Crash) bool { - for _, re := range mgr.cfg.ParsedSuppressions { - if !re.Match(crash.Output) { - continue - } - log.Logf(0, "vm-%v: suppressing '%v' with '%v'", crash.vmIndex, crash.Title, re.String()) - mgr.mu.Lock() - mgr.stats["suppressed"]++ - mgr.mu.Unlock() - return true - } - return false -} - func (mgr *Manager) emailCrash(crash *Crash) { if len(mgr.cfg.EmailAddrs) == 0 { return @@ -628,12 +616,19 @@ func (mgr *Manager) emailCrash(crash *Crash) { } func (mgr *Manager) saveCrash(crash *Crash) bool { + if crash.Suppressed { + log.Logf(0, "vm-%v: suppressed crash %v", crash.vmIndex, crash.Title) + mgr.mu.Lock() + mgr.stats["suppressed"]++ + mgr.mu.Unlock() + return false + } corrupted := "" if crash.Corrupted { corrupted = " [corrupted]" } log.Logf(0, "vm-%v: crash: %v%v", crash.vmIndex, crash.Title, corrupted) - if err := mgr.getReporter().Symbolize(crash.Report); err != nil { + if err := mgr.reporter.Symbolize(crash.Report); err != nil { log.Logf(0, "failed to symbolize report: %v", err) } @@ -743,7 +738,7 @@ func (mgr *Manager) saveFailedRepro(desc string) { func (mgr *Manager) saveRepro(res *repro.Result, hub bool) { rep := res.Report - if err := mgr.getReporter().Symbolize(rep); err != nil { + if err := mgr.reporter.Symbolize(rep); err != nil { log.Logf(0, "failed to symbolize repro: %v", err) } opts := fmt.Sprintf("# %+v\n", res.Opts) @@ -824,26 +819,6 @@ func (mgr *Manager) saveRepro(res *repro.Result, hub bool) { osutil.WriteFile(filepath.Join(dir, "repro.stats"), []byte(stats)) } -func (mgr *Manager) getReporter() report.Reporter { - mgr.reporterInit.Do(func() { - <-allSymbolsReady - var err error - // TODO(dvyukov): we should introduce cfg.Kernel_Obj dir instead of Vmlinux. - // This will be more general taking into account modules and other OSes. - kernelSrc, kernelObj := "", "" - if mgr.cfg.Vmlinux != "" { - kernelSrc = mgr.cfg.KernelSrc - kernelObj = filepath.Dir(mgr.cfg.Vmlinux) - } - mgr.reporter, err = report.NewReporter(mgr.cfg.TargetOS, mgr.cfg.Type, - kernelSrc, kernelObj, allSymbols, mgr.cfg.ParsedIgnores) - if err != nil { - log.Fatalf("%v", err) - } - }) - return mgr.reporter -} - func (mgr *Manager) minimizeCorpus() { if mgr.phase < phaseLoadedCorpus { return @@ -1240,7 +1215,9 @@ func (mgr *Manager) collectUsedFiles() { addUsedFile(cfg.SyzExecprogBin) addUsedFile(cfg.SyzExecutorBin) addUsedFile(cfg.SSHKey) - addUsedFile(cfg.Vmlinux) + if vmlinux := filepath.Join(cfg.KernelObj, "vmlinux"); osutil.IsExist(vmlinux) { + addUsedFile(vmlinux) + } if cfg.Image != "9p" { addUsedFile(cfg.Image) } diff --git a/syz-manager/mgrconfig/mgrconfig.go b/syz-manager/mgrconfig/mgrconfig.go index 7e671a13..c7057246 100644 --- a/syz-manager/mgrconfig/mgrconfig.go +++ b/syz-manager/mgrconfig/mgrconfig.go @@ -8,7 +8,6 @@ import ( "fmt" "os" "path/filepath" - "regexp" "strings" "github.com/google/syzkaller/pkg/config" @@ -16,7 +15,6 @@ import ( "github.com/google/syzkaller/prog" _ "github.com/google/syzkaller/sys" // most mgrconfig users want targets too "github.com/google/syzkaller/sys/targets" - "github.com/google/syzkaller/vm" ) type Config struct { @@ -27,9 +25,12 @@ type Config struct { // TCP address to serve HTTP stats page (e.g. "localhost:50000"). HTTP string `json:"http"` // TCP address to serve RPC for fuzzer processes (optional). - RPC string `json:"rpc"` - Workdir string `json:"workdir"` - Vmlinux string `json:"vmlinux"` + RPC string `json:"rpc"` + Workdir string `json:"workdir"` + VmlinuxUnused string `json:"vmlinux"` // vmlinux should go away eventually. + // Directory with kernel object files. + // If not set, inferred as base dir of Vmlinux. + KernelObj string `json:"kernel_obj"` // Kernel source directory. KernelSrc string `json:"kernel_src"` // Arbitrary optional tag that is saved along with crash reports (e.g. branch/commit). @@ -72,9 +73,11 @@ type Config struct { EnabledSyscalls []string `json:"enable_syscalls"` DisabledSyscalls []string `json:"disable_syscalls"` - // Don't save reports matching these regexps, but reboot VM after them. + // Don't save reports matching these regexps, but reboot VM after them, + // matched against whole report output. Suppressions []string `json:"suppressions"` - // Completely ignore reports matching these regexps (don't save nor reboot). + // Completely ignore reports matching these regexps (don't save nor reboot), + // must match the first line of crash message. Ignores []string `json:"ignores"` // VM type (qemu, gce, android, isolated, etc). @@ -83,8 +86,6 @@ type Config struct { VM json.RawMessage `json:"vm"` // Implementation details beyond this point. - ParsedSuppressions []*regexp.Regexp `json:"-"` - ParsedIgnores []*regexp.Regexp `json:"-"` // Parsed Target: TargetOS string `json:"-"` TargetArch string `json:"-"` @@ -208,15 +209,13 @@ func Complete(cfg *Config) error { } } - cfg.Vmlinux = osutil.Abs(cfg.Vmlinux) + cfg.VmlinuxUnused = osutil.Abs(cfg.VmlinuxUnused) + if cfg.KernelObj == "" { + cfg.KernelObj = filepath.Dir(cfg.VmlinuxUnused) // assume in-tree build by default + } if cfg.KernelSrc == "" { - cfg.KernelSrc = filepath.Dir(cfg.Vmlinux) // assume in-tree build by default + cfg.KernelSrc = filepath.Dir(cfg.VmlinuxUnused) // assume in-tree build by default } - - if err := parseSuppressions(cfg); err != nil { - return err - } - if cfg.HubClient != "" && (cfg.Name == "" || cfg.HubAddr == "" || cfg.HubKey == "") { return fmt.Errorf("hub_client is set, but name/hub_addr/hub_key is empty") } @@ -294,54 +293,3 @@ func matchSyscall(name, pattern string) bool { } return false } - -func parseSuppressions(cfg *Config) error { - // Add some builtin suppressions. - // TODO(dvyukov): this should be moved to pkg/report. - supp := append(cfg.Suppressions, []string{ - "panic: failed to start executor binary", - "panic: executor failed: pthread_create failed", - "panic: failed to create temp dir", - "fatal error: runtime: out of memory", - "fatal error: runtime: cannot allocate memory", - "fatal error: unexpected signal during runtime execution", // presubmably OOM turned into SIGBUS - "signal SIGBUS: bus error", // presubmably OOM turned into SIGBUS - // TODO(dvyukov): these should be moved sys/targets as they are really linux-specific. - "Out of memory: Kill process .* \\(syz-fuzzer\\)", - "Out of memory: Kill process .* \\(sshd\\)", - "Killed process .* \\(syz-fuzzer\\)", - "Killed process .* \\(sshd\\)", - "lowmemorykiller: Killing 'syz-fuzzer'", - "lowmemorykiller: Killing 'sshd'", - "INIT: PANIC: segmentation violation!", - }...) - for _, s := range supp { - re, err := regexp.Compile(s) - if err != nil { - return fmt.Errorf("failed to compile suppression '%v': %v", s, err) - } - cfg.ParsedSuppressions = append(cfg.ParsedSuppressions, re) - } - for _, ignore := range cfg.Ignores { - re, err := regexp.Compile(ignore) - if err != nil { - return fmt.Errorf("failed to compile ignore '%v': %v", ignore, err) - } - cfg.ParsedIgnores = append(cfg.ParsedIgnores, re) - } - return nil -} - -func CreateVMEnv(cfg *Config, debug bool) *vm.Env { - return &vm.Env{ - Name: cfg.Name, - OS: cfg.TargetOS, - Arch: cfg.TargetVMArch, - Workdir: cfg.Workdir, - Image: cfg.Image, - SSHKey: cfg.SSHKey, - SSHUser: cfg.SSHUser, - Debug: debug, - Config: cfg.VM, - } -} diff --git a/tools/syz-crush/crush.go b/tools/syz-crush/crush.go index b73c9027..bb07b72c 100644 --- a/tools/syz-crush/crush.go +++ b/tools/syz-crush/crush.go @@ -10,7 +10,6 @@ import ( "flag" "fmt" "io/ioutil" - "path/filepath" "sync" "sync/atomic" "time" @@ -39,13 +38,11 @@ func main() { if _, err := prog.GetTarget(cfg.TargetOS, cfg.TargetArch); err != nil { log.Fatalf("%v", err) } - env := mgrconfig.CreateVMEnv(cfg, false) - vmPool, err := vm.Create(cfg.Type, env) + vmPool, err := vm.Create(cfg, false) if err != nil { log.Fatalf("%v", err) } - reporter, err := report.NewReporter(cfg.TargetOS, cfg.Type, - cfg.KernelSrc, filepath.Dir(cfg.Vmlinux), nil, cfg.ParsedIgnores) + reporter, err := report.NewReporter(cfg) if err != nil { log.Fatalf("%v", err) } diff --git a/tools/syz-parse/syz-parse.go b/tools/syz-parse/syz-parse.go deleted file mode 100644 index 64841dab..00000000 --- a/tools/syz-parse/syz-parse.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2017 syzkaller project authors. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -package main - -import ( - "fmt" - "io/ioutil" - "os" - - "github.com/google/syzkaller/pkg/report" -) - -func main() { - if len(os.Args) < 2 { - usage() - return - } - switch os.Args[1] { - case "report": - if len(os.Args) != 4 { - usage() - return - } - parseReport(os.Args[2], os.Args[3]) - default: - usage() - } -} - -func usage() { - fmt.Fprintf(os.Stderr, "usage:\n") - fmt.Fprintf(os.Stderr, " syz-parse report \n") - os.Exit(1) -} - -func parseReport(os, file string) { - log, err := ioutil.ReadFile(file) - if err != nil { - fmt.Printf("Error: %v\n", err) - return - } - reporter, err := report.NewReporter(os, "", "", "", nil, nil) - if err != nil { - fmt.Printf("Error: %v\n", err) - return - } - rep := reporter.Parse(log) - if rep == nil { - fmt.Printf("Couldn't find any reports\n") - return - } - fmt.Printf("=======\n") - fmt.Printf("Title: %v\n", rep.Title) - fmt.Printf("Corrupted: %v\n", rep.Corrupted) - fmt.Printf("Report:\n%s\n", rep.Report) -} diff --git a/tools/syz-repro/repro.go b/tools/syz-repro/repro.go index 8fcb04e4..84e2d96f 100644 --- a/tools/syz-repro/repro.go +++ b/tools/syz-repro/repro.go @@ -41,8 +41,7 @@ func main() { if _, err := prog.GetTarget(cfg.TargetOS, cfg.TargetArch); err != nil { log.Fatalf("%v", err) } - env := mgrconfig.CreateVMEnv(cfg, false) - vmPool, err := vm.Create(cfg.Type, env) + vmPool, err := vm.Create(cfg, false) if err != nil { log.Fatalf("%v", err) } @@ -57,7 +56,7 @@ func main() { for i := range vmIndexes { vmIndexes[i] = i } - reporter, err := report.NewReporter(cfg.TargetOS, cfg.Type, cfg.KernelSrc, "", nil, cfg.ParsedIgnores) + reporter, err := report.NewReporter(cfg) if err != nil { log.Fatalf("%v", err) } diff --git a/tools/syz-symbolize/symbolize.go b/tools/syz-symbolize/symbolize.go index e683a5b0..9020d2fc 100644 --- a/tools/syz-symbolize/symbolize.go +++ b/tools/syz-symbolize/symbolize.go @@ -11,6 +11,7 @@ import ( "runtime" "github.com/google/syzkaller/pkg/report" + "github.com/google/syzkaller/syz-manager/mgrconfig" ) var ( @@ -26,7 +27,12 @@ func main() { flag.PrintDefaults() os.Exit(1) } - reporter, err := report.NewReporter(*flagOS, "", *flagKernelSrc, *flagKernelObj, nil, nil) + cfg := &mgrconfig.Config{ + TargetOS: *flagOS, + KernelObj: *flagKernelObj, + KernelSrc: *flagKernelSrc, + } + reporter, err := report.NewReporter(cfg) if err != nil { fmt.Fprintf(os.Stderr, "failed to create reporter: %v\n", err) os.Exit(1) @@ -44,5 +50,8 @@ func main() { fmt.Fprintf(os.Stderr, "failed to symbolize report: %v\n", err) os.Exit(1) } + fmt.Printf("TITLE: %v\n", rep.Title) + fmt.Printf("CORRUPTED: %v\n", rep.Corrupted) + fmt.Printf("\n") os.Stdout.Write(rep.Report) } diff --git a/vm/vm.go b/vm/vm.go index de6071cf..a990f0cf 100644 --- a/vm/vm.go +++ b/vm/vm.go @@ -16,6 +16,7 @@ import ( "github.com/google/syzkaller/pkg/osutil" "github.com/google/syzkaller/pkg/report" + "github.com/google/syzkaller/syz-manager/mgrconfig" "github.com/google/syzkaller/vm/vmimpl" // Import all VM implementations, so that users only need to import vm. @@ -39,8 +40,6 @@ type Instance struct { index int } -type Env vmimpl.Env - var ( Shutdown = vmimpl.Shutdown ErrTimeout = vmimpl.ErrTimeout @@ -50,8 +49,19 @@ type BootErrorer interface { BootError() (string, []byte) } -func Create(typ string, env *Env) (*Pool, error) { - impl, err := vmimpl.Create(typ, (*vmimpl.Env)(env)) +func Create(cfg *mgrconfig.Config, debug bool) (*Pool, error) { + env := &vmimpl.Env{ + Name: cfg.Name, + OS: cfg.TargetOS, + Arch: cfg.TargetVMArch, + Workdir: cfg.Workdir, + Image: cfg.Image, + SSHKey: cfg.SSHKey, + SSHUser: cfg.SSHUser, + Debug: debug, + Config: cfg.VM, + } + impl, err := vmimpl.Create(cfg.Type, env) if err != nil { return nil, err }