diff --git a/pkg/cover/cover.go b/pkg/cover/cover.go index 4913d1e2..367f68d4 100644 --- a/pkg/cover/cover.go +++ b/pkg/cover/cover.go @@ -1,182 +1,30 @@ // Copyright 2015 syzkaller project authors. All rights reserved. // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. -/* Package cover implements set operations on slices (arrays of coverage PCs). */ +// Package cover provides types for working with coverage information (arrays of covered PCs). package cover -import ( - "sort" -) +type Cover map[uint32]struct{} -type Cover []uint32 +func (covp *Cover) Merge(raw []uint32) { + cov := *covp + if cov == nil { + cov = make(Cover) + *covp = cov + } + for _, pc := range raw { + cov[pc] = struct{}{} + } +} -func (a Cover) Len() int { return len(a) } -func (a Cover) Less(i, j int) bool { return a[i] < a[j] } -func (a Cover) Swap(i, j int) { a[i], a[j] = a[j], a[i] } - -const sent = ^uint32(0) - -func Copy(cov Cover) Cover { - return append(Cover{}, cov...) +func (cov Cover) Serialize() []uint32 { + res := make([]uint32, 0, len(cov)) + for pc := range cov { + res = append(res, pc) + } + return res } func RestorePC(pc uint32, base uint32) uint64 { return uint64(base)<<32 + uint64(pc) } - -// Canonicalize sorts and removes duplicates. -func Canonicalize(cov []uint32) Cover { - sort.Sort(Cover(cov)) - i := 0 - last := sent - for _, pc := range cov { - if pc != last { - last = pc - cov[i] = pc - i++ - } - } - return Cover(cov[:i]) -} - -func Difference(cov0, cov1 Cover) Cover { - return foreach(cov0, cov1, func(v0, v1 uint32) uint32 { - if v0 < v1 { - return v0 - } - return sent - }) -} - -func SymmetricDifference(cov0, cov1 Cover) Cover { - return foreach(cov0, cov1, func(v0, v1 uint32) uint32 { - if v0 < v1 { - return v0 - } - if v1 < v0 { - return v1 - } - return sent - }) -} - -func Union(cov0, cov1 Cover) Cover { - return foreach(cov0, cov1, func(v0, v1 uint32) uint32 { - if v0 <= v1 { - return v0 - } - return v1 - }) -} - -func Intersection(cov0, cov1 Cover) Cover { - return foreach(cov0, cov1, func(v0, v1 uint32) uint32 { - if v0 == v1 { - return v0 - } - return sent - }) -} - -func foreach(cov0, cov1 Cover, f func(uint32, uint32) uint32) Cover { - var res []uint32 - for i0, i1 := 0, 0; i0 < len(cov0) || i1 < len(cov1); { - v0, v1 := sent, sent - if i0 < len(cov0) { - v0 = cov0[i0] - } - if i1 < len(cov1) { - v1 = cov1[i1] - } - if v0 <= v1 { - i0++ - } - if v1 <= v0 { - i1++ - } - if v := f(v0, v1); v != sent { - res = append(res, v) - } - } - return res -} - -// HasDifference returns true if cov0 has some coverage that is not present in cov1. -// This is called on fuzzer hot path. -func HasDifference(cov0, cov1 Cover) bool { - i1 := 0 - for _, v0 := range cov0 { - for ; i1 < len(cov1) && cov1[i1] < v0; i1++ { - } - if i1 == len(cov1) || cov1[i1] > v0 { - return true - } - i1++ - } - return false -} - -// Minimize returns a minimal set of inputs that give the same coverage as the full corpus. -func Minimize(corpus []Cover) []int { - inputs := make([]*minInput, len(corpus)) - for i, cov := range corpus { - inputs[i] = &minInput{ - idx: i, - cov: cov, - } - } - sort.Sort(minInputArray(inputs)) - var min []int - covered := make(map[uint32]struct{}) - for _, inp := range inputs { - hit := false - for _, pc := range inp.cov { - if !hit { - if _, ok := covered[pc]; !ok { - hit = true - min = append(min, inp.idx) - } - } - if hit { - covered[pc] = struct{}{} - } - } - } - return min -} - -type minInput struct { - idx int - cov Cover -} - -type minInputArray []*minInput - -// Inputs with larger coverage come first. -func (a minInputArray) Len() int { return len(a) } -func (a minInputArray) Less(i, j int) bool { return len(a[i].cov) > len(a[j].cov) } -func (a minInputArray) Swap(i, j int) { a[i], a[j] = a[j], a[i] } - -func SignalNew(base map[uint32]struct{}, signal []uint32) bool { - for _, s := range signal { - if _, ok := base[s]; !ok { - return true - } - } - return false -} - -func SignalDiff(base map[uint32]struct{}, signal []uint32) (diff []uint32) { - for _, s := range signal { - if _, ok := base[s]; !ok { - diff = append(diff, s) - } - } - return -} - -func SignalAdd(base map[uint32]struct{}, signal []uint32) { - for _, s := range signal { - base[s] = struct{}{} - } -} diff --git a/pkg/cover/cover_test.go b/pkg/cover/cover_test.go deleted file mode 100644 index 5297de66..00000000 --- a/pkg/cover/cover_test.go +++ /dev/null @@ -1,235 +0,0 @@ -// Copyright 2015 syzkaller project authors. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -package cover - -import ( - "math/rand" - "reflect" - "sort" - "testing" - "time" -) - -func initTest(t *testing.T) (*rand.Rand, int) { - iters := 100000 - if testing.Short() { - iters = 1000 - } - seed := int64(time.Now().UnixNano()) - rs := rand.NewSource(seed) - t.Logf("seed=%v", seed) - return rand.New(rs), iters -} - -type Test struct { - V0 Cover - V1 Cover - R Cover -} - -func runTest(t *testing.T, f func(Cover, Cover) Cover, sorted, symmetric bool, tests []Test) { - if symmetric { - for _, test := range tests { - tests = append(tests, Test{test.V1, test.V0, test.R}) - } - } - tests = append(tests, Test{Cover{}, Cover{}, Cover{}}) - for _, test := range tests { - if sorted { - if !sort.IsSorted(test.V0) { - t.Fatalf("input is not sorted: %+v", test.V0) - } - if !sort.IsSorted(test.V1) { - t.Fatalf("input is not sorted: %+v", test.V1) - } - } - if !sort.IsSorted(test.R) { - t.Fatalf("golden is not sorted: %+v", test.R) - } - res := f(test.V0, test.V1) - if !sort.IsSorted(res) { - t.Fatalf("output is not sorted: %+v", res) - } - if (len(res) != 0 || len(test.R) != 0) && !reflect.DeepEqual(res, test.R) { - t.Fatalf("f(%+v, %+v) = %+v (expect: %+v)", test.V0, test.V1, res, test.R) - } - } -} - -func TestCanonicalize(t *testing.T) { - runTest(t, func(c0, c1 Cover) Cover { return Canonicalize([]uint32(c0)) }, false, false, []Test{ - {Cover{1, 1, 2, 3, 3, 4, 5, 5, 5, 6, 6}, Cover{}, Cover{1, 2, 3, 4, 5, 6}}, - {Cover{6, 2, 3, 4, 5, 1}, Cover{}, Cover{1, 2, 3, 4, 5, 6}}, - {Cover{6, 1, 2, 6, 3, 3, 4, 5, 1}, Cover{}, Cover{1, 2, 3, 4, 5, 6}}, - }) -} - -func TestDifference(t *testing.T) { - runTest(t, Difference, true, false, []Test{ - {Cover{1, 2, 3, 4, 5, 6}, Cover{}, Cover{1, 2, 3, 4, 5, 6}}, - {Cover{1, 2, 3, 4, 5, 6}, Cover{3}, Cover{1, 2, 4, 5, 6}}, - {Cover{1, 2, 3, 4, 5, 6}, Cover{1, 6}, Cover{2, 3, 4, 5}}, - {Cover{1, 2, 3, 4, 5, 6}, Cover{0, 10}, Cover{1, 2, 3, 4, 5, 6}}, - {Cover{1, 2, 3, 4, 5, 6}, Cover{0, 3, 6}, Cover{1, 2, 4, 5}}, - }) -} - -func TestSymmetricDifference(t *testing.T) { - runTest(t, SymmetricDifference, true, true, []Test{ - {Cover{1, 2, 3, 4, 5, 6}, Cover{}, Cover{1, 2, 3, 4, 5, 6}}, - {Cover{1, 2, 3, 4, 5, 6}, Cover{1, 2, 3, 4, 5, 6}, Cover{}}, - {Cover{1, 2, 3, 4, 5, 6}, Cover{2, 4, 6}, Cover{1, 3, 5}}, - {Cover{2, 4, 6}, Cover{1, 3, 5}, Cover{1, 2, 3, 4, 5, 6}}, - }) -} - -func TestUnion(t *testing.T) { - runTest(t, Union, true, true, []Test{ - {Cover{1, 2, 3, 4, 5, 6}, Cover{}, Cover{1, 2, 3, 4, 5, 6}}, - {Cover{1, 2, 3, 4, 5, 6}, Cover{1, 2, 3, 4, 5, 6}, Cover{1, 2, 3, 4, 5, 6}}, - {Cover{1, 3, 5}, Cover{2, 4, 6}, Cover{1, 2, 3, 4, 5, 6}}, - {Cover{1, 2, 3, 5}, Cover{2, 4, 5, 6}, Cover{1, 2, 3, 4, 5, 6}}, - }) -} - -func TestIntersection(t *testing.T) { - runTest(t, Intersection, true, true, []Test{ - {Cover{1, 2, 3, 4, 5, 6}, Cover{}, Cover{}}, - {Cover{1, 2, 3}, Cover{4, 5, 6}, Cover{}}, - {Cover{1, 2, 3}, Cover{2, 3, 5}, Cover{2, 3}}, - }) -} - -func TestMinimize(t *testing.T) { - tests := []struct { - inp []Cover - out []int - }{ - // Take all. - { - []Cover{ - {1, 2, 3}, - {4, 5, 6}, - {7, 8, 9}, - }, - []int{0, 1, 2}, - }, - // Take one. - { - []Cover{ - {1, 2, 3, 4, 5, 6, 7, 8, 9}, - {1}, - {2, 3, 4, 5}, - {6, 7, 8}, - }, - []int{0}, - }, - // Take two. - { - []Cover{ - {1, 2, 3, 4, 5, 6, 7, 8, 9}, - {1}, - {2, 3, 4, 5}, - {10}, - }, - []int{0, 3}, - }, - // Take another two. - { - []Cover{ - {1, 2, 3, 4}, - {1, 2}, - {3, 4, 5, 6, 7}, - {3, 7}, - }, - []int{2, 0}, - }, - // Take the largest one. - { - []Cover{ - {1, 2}, - {1, 2, 3, 4, 5}, - {3, 4, 5}, - }, - []int{1}, - }, - } - for _, test := range tests { - res := Minimize(test.inp) - if !reflect.DeepEqual(res, test.out) { - t.Logf("corpus:") - for _, in := range test.inp { - t.Logf(" %+v", in) - } - t.Fatalf("expect: %+v, got: %+v", test.out, res) - } - } -} - -func randCover(rnd *rand.Rand, maxLen int) Cover { - tmp := make(Cover, rnd.Intn(maxLen)) - for j := range tmp { - tmp[j] = uint32(rnd.Intn(100)) - } - return Canonicalize(tmp) -} - -func TestMinimizeRandom(t *testing.T) { - rnd, iters := initTest(t) - for i := 0; i < iters; i++ { - n := rnd.Intn(20) - cov := make([]Cover, n) - for i := range cov { - cov[i] = randCover(rnd, 10) - } - var total Cover - for _, c := range cov { - total = Union(total, c) - } - mini := Minimize(cov) - var minimized Cover - for _, idx := range mini { - minimized = Union(minimized, cov[idx]) - } - if !reflect.DeepEqual(total, minimized) { - t.Logf("minimized %v -> %v", len(cov), len(mini)) - t.Logf("corpus:") - for _, in := range cov { - t.Logf(" %+v", in) - } - t.Logf("minimized:") - for _, in := range cov { - t.Logf(" %+v", in) - } - t.Fatalf("better luck next time") - } - } -} - -func TestHasDifference(t *testing.T) { - rnd, iters := initTest(t) - for i := 0; i < iters; i++ { - cov1 := randCover(rnd, 20) - cov2 := randCover(rnd, 20) - diff := Difference(cov1, cov2) - hasDiff := HasDifference(cov1, cov2) - if len(diff) != 0 != hasDiff { - t.Fatalf("cov1=%+v cov2=%+v diff=%+v hasDiff=%v", cov1, cov2, diff, hasDiff) - } - } -} - -func BenchmarkHasDifference(b *testing.B) { - rnd := rand.New(rand.NewSource(0)) - cov0 := make(Cover, 70000) - for i := range cov0 { - cov0[i] = uint32(rnd.Intn(1 << 30)) - } - cov1 := Canonicalize(append(Cover{}, cov0[:500]...)) - cov0 = Canonicalize(cov0) - b.ResetTimer() - for i := 0; i < b.N; i++ { - _ = HasDifference(cov1, cov0) - } -} diff --git a/pkg/rpctype/rpctype.go b/pkg/rpctype/rpctype.go index e4d62e2f..d7fee32d 100644 --- a/pkg/rpctype/rpctype.go +++ b/pkg/rpctype/rpctype.go @@ -5,10 +5,14 @@ // between various parts of the system. package rpctype +import ( + "github.com/google/syzkaller/pkg/signal" +) + type RpcInput struct { Call string Prog []byte - Signal []uint32 + Signal signal.Serial Cover []uint32 } @@ -25,7 +29,7 @@ type ConnectArgs struct { type ConnectRes struct { Prios [][]float32 Inputs []RpcInput - MaxSignal []uint32 + MaxSignal signal.Serial Candidates []RpcCandidate EnabledCalls string NeedCheck bool @@ -54,14 +58,14 @@ type NewInputArgs struct { type PollArgs struct { Name string NeedCandidates bool - MaxSignal []uint32 + MaxSignal signal.Serial Stats map[string]uint64 } type PollRes struct { Candidates []RpcCandidate NewInputs []RpcInput - MaxSignal []uint32 + MaxSignal signal.Serial } type HubConnectArgs struct { diff --git a/pkg/signal/signal.go b/pkg/signal/signal.go new file mode 100644 index 00000000..366407d0 --- /dev/null +++ b/pkg/signal/signal.go @@ -0,0 +1,166 @@ +// Copyright 2018 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +// Package signal provides types for working with feedback signal. +package signal + +import ( + "sort" +) + +type ( + elemType uint32 + prioType int8 +) + +type Signal map[elemType]prioType + +type Serial struct { + Elems []elemType + Prios []prioType +} + +func (s Signal) Len() int { + return len(s) +} + +func (s Signal) Empty() bool { + return len(s) == 0 +} + +func FromRaw(raw []uint32, prio uint8) Signal { + if len(raw) == 0 { + return nil + } + s := make(Signal, len(raw)) + for _, e := range raw { + s[elemType(e)] = prioType(prio) + } + return s +} + +func (s Signal) Serialize() Serial { + if s.Empty() { + return Serial{} + } + res := Serial{ + Elems: make([]elemType, len(s)), + Prios: make([]prioType, len(s)), + } + i := 0 + for e, p := range s { + res.Elems[i] = e + res.Prios[i] = p + i++ + } + return res +} + +func (ser Serial) Deserialize() Signal { + if len(ser.Elems) != len(ser.Prios) { + panic("corrupted Serial") + } + if len(ser.Elems) == 0 { + return nil + } + s := make(Signal, len(ser.Elems)) + for i, e := range ser.Elems { + s[e] = ser.Prios[i] + } + return s +} + +func (s Signal) Diff(s1 Signal) Signal { + if s1.Empty() { + return nil + } + var res Signal + for e, p1 := range s1 { + if p, ok := s[e]; ok && p >= p1 { + continue + } + if res == nil { + res = make(Signal) + } + res[e] = p1 + } + return res +} + +func (s Signal) DiffRaw(raw []uint32, prio uint8) Signal { + var res Signal + for _, e := range raw { + if p, ok := s[elemType(e)]; ok && p >= prioType(prio) { + continue + } + if res == nil { + res = make(Signal) + } + res[elemType(e)] = prioType(prio) + } + return res +} + +func (s Signal) Intersection(s1 Signal) Signal { + if s1.Empty() { + return nil + } + res := make(Signal, len(s)) + for e, p := range s { + if p1, ok := s1[e]; ok && p1 >= p { + res[e] = p + } + } + return res +} + +func (sp *Signal) Merge(s1 Signal) { + if s1.Empty() { + return + } + s := *sp + if s == nil { + s = make(Signal, len(s1)) + *sp = s + } + for e, p1 := range s1 { + if p, ok := s[e]; !ok || p < p1 { + s[e] = p1 + } + } +} + +type SignalContext struct { + Signal Signal + Context interface{} +} + +func Minimize(corpus []SignalContext) []interface{} { + sort.Slice(corpus, func(i, j int) bool { + return corpus[i].Signal.Len() > corpus[j].Signal.Len() + }) + type ContextPrio struct { + prio prioType + idx int + } + covered := make(map[elemType]ContextPrio) + for i, inp := range corpus { + for e, p := range inp.Signal { + if prev, ok := covered[e]; !ok || p > prev.prio { + covered[e] = ContextPrio{ + prio: p, + idx: i, + } + } + } + } + indices := make(map[int]struct{}, len(corpus)) + for _, cp := range covered { + indices[cp.idx] = struct{}{} + } + result := make([]interface{}, 0, len(indices)) + for idx := range indices { + result = append(result, corpus[idx].Context) + } + return result +} diff --git a/syz-fuzzer/fuzzer.go b/syz-fuzzer/fuzzer.go index f3befcd3..9f045338 100644 --- a/syz-fuzzer/fuzzer.go +++ b/syz-fuzzer/fuzzer.go @@ -18,13 +18,13 @@ import ( "syscall" "time" - "github.com/google/syzkaller/pkg/cover" "github.com/google/syzkaller/pkg/hash" "github.com/google/syzkaller/pkg/host" "github.com/google/syzkaller/pkg/ipc" . "github.com/google/syzkaller/pkg/log" "github.com/google/syzkaller/pkg/osutil" . "github.com/google/syzkaller/pkg/rpctype" + "github.com/google/syzkaller/pkg/signal" "github.com/google/syzkaller/prog" "github.com/google/syzkaller/sys" ) @@ -54,9 +54,9 @@ type Fuzzer struct { corpusHashes map[hash.Sig]struct{} signalMu sync.RWMutex - corpusSignal map[uint32]struct{} // coverage of inputs in corpus - maxSignal map[uint32]struct{} // max coverage ever observed including flakes - newSignal map[uint32]struct{} // diff of maxSignal since last sync with master + corpusSignal signal.Signal // signal of inputs in corpus + maxSignal signal.Signal // max signal ever observed including flakes + newSignal signal.Signal // diff of maxSignal since last sync with master logMu sync.Mutex } @@ -245,16 +245,13 @@ func main() { coverageEnabled: coverageEnabled, leakCheckEnabled: *flagLeak, corpusHashes: make(map[hash.Sig]struct{}), - corpusSignal: make(map[uint32]struct{}), - maxSignal: make(map[uint32]struct{}), - newSignal: make(map[uint32]struct{}), } fuzzer.gate = ipc.NewGate(2**flagProcs, fuzzer.leakCheckCallback) for _, inp := range r.Inputs { fuzzer.addInputFromAnotherFuzzer(inp) } - fuzzer.addMaxSignal(r.MaxSignal) + fuzzer.addMaxSignal(r.MaxSignal.Deserialize()) for _, candidate := range r.Candidates { p, err := fuzzer.target.Deserialize(candidate.Prog) if err != nil { @@ -317,7 +314,7 @@ func (fuzzer *Fuzzer) pollLoop() { NeedCandidates: needCandidates, Stats: make(map[string]uint64), } - a.MaxSignal = fuzzer.grabNewSignal() + a.MaxSignal = fuzzer.grabNewSignal().Serialize() for _, proc := range fuzzer.procs { a.Stats["exec total"] += atomic.SwapUint64(&proc.env.StatExecs, 0) a.Stats["executor restarts"] += atomic.SwapUint64(&proc.env.StatRestarts, 0) @@ -333,9 +330,10 @@ func (fuzzer *Fuzzer) pollLoop() { if err := fuzzer.manager.Call("Manager.Poll", a, r); err != nil { panic(err) } + maxSignal := r.MaxSignal.Deserialize() Logf(1, "poll: candidates=%v inputs=%v signal=%v", - len(r.Candidates), len(r.NewInputs), len(r.MaxSignal)) - fuzzer.addMaxSignal(r.MaxSignal) + len(r.Candidates), len(r.NewInputs), maxSignal.Len()) + fuzzer.addMaxSignal(maxSignal) for _, inp := range r.NewInputs { fuzzer.addInputFromAnotherFuzzer(inp) } @@ -427,10 +425,12 @@ func (fuzzer *Fuzzer) addInputFromAnotherFuzzer(inp RpcInput) { if err != nil { panic(err) } - fuzzer.addInputToCorpus(p, inp.Signal, hash.Hash(inp.Prog)) + sig := hash.Hash(inp.Prog) + sign := inp.Signal.Deserialize() + fuzzer.addInputToCorpus(p, sign, sig) } -func (fuzzer *Fuzzer) addInputToCorpus(p *prog.Prog, signal []uint32, sig hash.Sig) { +func (fuzzer *Fuzzer) addInputToCorpus(p *prog.Prog, sign signal.Signal, sig hash.Sig) { fuzzer.corpusMu.Lock() if _, ok := fuzzer.corpusHashes[sig]; !ok { fuzzer.corpus = append(fuzzer.corpus, p) @@ -438,10 +438,12 @@ func (fuzzer *Fuzzer) addInputToCorpus(p *prog.Prog, signal []uint32, sig hash.S } fuzzer.corpusMu.Unlock() - fuzzer.signalMu.Lock() - cover.SignalAdd(fuzzer.corpusSignal, signal) - cover.SignalAdd(fuzzer.maxSignal, signal) - fuzzer.signalMu.Unlock() + if !sign.Empty() { + fuzzer.signalMu.Lock() + fuzzer.corpusSignal.Merge(sign) + fuzzer.maxSignal.Merge(sign) + fuzzer.signalMu.Unlock() + } } func (fuzzer *Fuzzer) corpusSnapshot() []*prog.Prog { @@ -450,65 +452,58 @@ func (fuzzer *Fuzzer) corpusSnapshot() []*prog.Prog { return fuzzer.corpus } -func (fuzzer *Fuzzer) addMaxSignal(signal []uint32) { - if len(signal) == 0 { +func (fuzzer *Fuzzer) addMaxSignal(sign signal.Signal) { + if sign.Len() == 0 { return } fuzzer.signalMu.Lock() defer fuzzer.signalMu.Unlock() - for _, s := range signal { - fuzzer.maxSignal[s] = struct{}{} - } + fuzzer.maxSignal.Merge(sign) } -func (fuzzer *Fuzzer) grabNewSignal() []uint32 { +func (fuzzer *Fuzzer) grabNewSignal() signal.Signal { fuzzer.signalMu.Lock() - newSignal := fuzzer.newSignal - if len(newSignal) == 0 { - fuzzer.signalMu.Unlock() + defer fuzzer.signalMu.Unlock() + sign := fuzzer.newSignal + if sign.Empty() { return nil } - fuzzer.newSignal = make(map[uint32]struct{}) - fuzzer.signalMu.Unlock() - - signal := make([]uint32, 0, len(newSignal)) - for s := range newSignal { - signal = append(signal, s) - } - return signal + fuzzer.newSignal = nil + return sign } -func (fuzzer *Fuzzer) corpusSignalDiff(signal []uint32) []uint32 { - fuzzer.signalMu.Lock() - defer fuzzer.signalMu.Unlock() - return cover.SignalDiff(fuzzer.corpusSignal, signal) -} - -func (fuzzer *Fuzzer) addCorpusSignal(signal []uint32) []uint32 { - fuzzer.signalMu.Lock() - defer fuzzer.signalMu.Unlock() - return cover.SignalDiff(fuzzer.corpusSignal, signal) +func (fuzzer *Fuzzer) corpusSignalDiff(sign signal.Signal) signal.Signal { + fuzzer.signalMu.RLock() + defer fuzzer.signalMu.RUnlock() + return fuzzer.corpusSignal.Diff(sign) } func (fuzzer *Fuzzer) checkNewSignal(info []ipc.CallInfo) (calls []int) { fuzzer.signalMu.RLock() defer fuzzer.signalMu.RUnlock() for i, inf := range info { - if !cover.SignalNew(fuzzer.maxSignal, inf.Signal) { + diff := fuzzer.maxSignal.DiffRaw(inf.Signal, signalPrio(&inf)) + if diff.Empty() { continue } calls = append(calls, i) - diff := cover.SignalDiff(fuzzer.maxSignal, inf.Signal) fuzzer.signalMu.RUnlock() fuzzer.signalMu.Lock() - cover.SignalAdd(fuzzer.maxSignal, diff) - cover.SignalAdd(fuzzer.newSignal, diff) + fuzzer.maxSignal.Merge(diff) + fuzzer.newSignal.Merge(diff) fuzzer.signalMu.Unlock() fuzzer.signalMu.RLock() } return } +func signalPrio(ci *ipc.CallInfo) uint8 { + if ci.Errno == 0 { + return 1 + } + return 0 +} + func (fuzzer *Fuzzer) leakCheckCallback() { if atomic.LoadUint32(&fuzzer.leakCheckReady) != 0 { // Scan for leaks once in a while (it is damn slow). diff --git a/syz-fuzzer/proc.go b/syz-fuzzer/proc.go index 605febeb..15ddec71 100644 --- a/syz-fuzzer/proc.go +++ b/syz-fuzzer/proc.go @@ -18,6 +18,7 @@ import ( "github.com/google/syzkaller/pkg/ipc" . "github.com/google/syzkaller/pkg/log" . "github.com/google/syzkaller/pkg/rpctype" + "github.com/google/syzkaller/pkg/signal" "github.com/google/syzkaller/prog" ) @@ -98,20 +99,17 @@ func (proc *Proc) loop() { func (proc *Proc) triageInput(item *WorkTriage) { Logf(1, "#%v: triaging type=%x", proc.pid, item.flags) - if !proc.fuzzer.coverageEnabled { panic("should not be called when coverage is disabled") } - newSignal := proc.fuzzer.corpusSignalDiff(item.info.Signal) - if len(newSignal) == 0 { + inputSignal := signal.FromRaw(item.info.Signal, signalPrio(&item.info)) + newSignal := proc.fuzzer.corpusSignalDiff(inputSignal) + if newSignal.Empty() { return } - newSignal = cover.Canonicalize(newSignal) - call := item.p.Calls[item.call].Meta - - Logf(3, "triaging input for %v (new signal=%v)", call.CallName, len(newSignal)) + Logf(3, "triaging input for %v (new signal=%v)", call.CallName, newSignal.Len()) var inputCover cover.Cover const ( signalRuns = 3 @@ -130,17 +128,14 @@ func (proc *Proc) triageInput(item *WorkTriage) { continue } inf := info[item.call] - newSignal = cover.Intersection(newSignal, cover.Canonicalize(inf.Signal)) + thisSignal := signal.FromRaw(inf.Signal, signalPrio(&inf)) + newSignal = newSignal.Intersection(thisSignal) // Without !minimized check manager starts losing some considerable amount // of coverage after each restart. Mechanics of this are not completely clear. - if len(newSignal) == 0 && item.flags&ProgMinimized == 0 { + if newSignal.Empty() && item.flags&ProgMinimized == 0 { return } - if len(inputCover) == 0 { - inputCover = append([]uint32{}, inf.Cover...) - } else { - inputCover = cover.Union(inputCover, inf.Cover) - } + inputCover.Merge(inf.Cover) } if item.flags&ProgMinimized == 0 { item.p, item.call = prog.Minimize(item.p, item.call, false, @@ -156,8 +151,8 @@ func (proc *Proc) triageInput(item *WorkTriage) { // Successful calls are much more valuable. return false } - signal := cover.Canonicalize(inf.Signal) - if len(cover.Intersection(newSignal, signal)) == len(newSignal) { + thisSignal := signal.FromRaw(inf.Signal, signalPrio(&inf)) + if newSignal.Intersection(thisSignal).Len() == newSignal.Len() { return true } } @@ -172,11 +167,11 @@ func (proc *Proc) triageInput(item *WorkTriage) { proc.fuzzer.sendInputToManager(RpcInput{ Call: call.CallName, Prog: data, - Signal: []uint32(cover.Canonicalize(item.info.Signal)), - Cover: []uint32(inputCover), + Signal: inputSignal.Serialize(), + Cover: inputCover.Serialize(), }) - proc.fuzzer.addInputToCorpus(item.p, item.info.Signal, sig) + proc.fuzzer.addInputToCorpus(item.p, inputSignal, sig) if item.flags&ProgSmashed == 0 { proc.fuzzer.workQueue.enqueue(&WorkSmash{item.p, item.call}) diff --git a/syz-manager/cover.go b/syz-manager/cover.go index 0048dfa7..182be303 100644 --- a/syz-manager/cover.go +++ b/syz-manager/cover.go @@ -91,7 +91,7 @@ func initAllCover(vmlinux string) { }() } -func generateCoverHtml(w io.Writer, vmlinux string, cov []uint32) error { +func generateCoverHtml(w io.Writer, vmlinux string, cov cover.Cover) error { if len(cov) == 0 { return fmt.Errorf("No coverage data available") } @@ -100,9 +100,9 @@ func generateCoverHtml(w io.Writer, vmlinux string, cov []uint32) error { if err != nil { return err } - pcs := make([]uint64, len(cov)) - for i, pc := range cov { - pcs[i] = cover.RestorePC(pc, base) - callLen + pcs := make([]uint64, 0, len(cov)) + for pc := range cov { + pcs = append(pcs, cover.RestorePC(pc, base)-callLen) } uncovered, err := uncoveredPcsInFuncs(vmlinux, pcs) if err != nil { diff --git a/syz-manager/html.go b/syz-manager/html.go index a42d75d7..de4aeb91 100644 --- a/syz-manager/html.go +++ b/syz-manager/html.go @@ -98,7 +98,7 @@ func (mgr *Manager) collectSummary(data *UISummaryData) (map[string]*CallCov, er data.Stats = append(data.Stats, UIStat{Name: "corpus", Value: fmt.Sprint(len(mgr.corpus))}) data.Stats = append(data.Stats, UIStat{Name: "triage queue", Value: fmt.Sprint(len(mgr.candidates))}) data.Stats = append(data.Stats, UIStat{Name: "cover", Value: fmt.Sprint(len(mgr.corpusCover)), Link: "/cover"}) - data.Stats = append(data.Stats, UIStat{Name: "signal", Value: fmt.Sprint(len(mgr.corpusSignal))}) + data.Stats = append(data.Stats, UIStat{Name: "signal", Value: fmt.Sprint(mgr.corpusSignal.Len())}) secs := uint64(1) if !mgr.firstConnect.IsZero() { @@ -129,7 +129,7 @@ func (mgr *Manager) collectSummary(data *UISummaryData) (map[string]*CallCov, er } cc := calls[inp.Call] cc.count++ - cc.cov = cover.Union(cc.cov, cover.Cover(inp.Cover)) + cc.cov.Merge(inp.Cover) } return calls, nil @@ -188,12 +188,12 @@ func (mgr *Manager) httpCover(w http.ResponseWriter, r *http.Request) { } var cov cover.Cover if sig := r.FormValue("input"); sig != "" { - cov = mgr.corpus[sig].Cover + cov.Merge(mgr.corpus[sig].Cover) } else { call := r.FormValue("call") for _, inp := range mgr.corpus { if call == "" || call == inp.Call { - cov = cover.Union(cov, cover.Cover(inp.Cover)) + cov.Merge(inp.Cover) } } } @@ -305,14 +305,20 @@ func (mgr *Manager) httpRawCover(w http.ResponseWriter, r *http.Request) { var cov cover.Cover for _, inp := range mgr.corpus { - cov = cover.Union(cov, cover.Cover(inp.Cover)) + cov.Merge(inp.Cover) } + pcs := make([]uint64, 0, len(cov)) + for pc := range cov { + pcs = append(pcs, cover.RestorePC(pc, base)-callLen) + } + sort.Slice(pcs, func(i, j int) bool { + return pcs[i] < pcs[j] + }) w.Header().Set("Content-Type", "text/plain; charset=utf-8") buf := bufio.NewWriter(w) - for _, pc := range cov { - restored := cover.RestorePC(pc, base) - callLen - fmt.Fprintf(buf, "0x%x\n", restored) + for _, pc := range pcs { + fmt.Fprintf(buf, "0x%x\n", pc) } buf.Flush() } diff --git a/syz-manager/manager.go b/syz-manager/manager.go index 7b4c5c93..923a9898 100644 --- a/syz-manager/manager.go +++ b/syz-manager/manager.go @@ -12,11 +12,9 @@ import ( "net" "os" "os/exec" - "os/signal" "path/filepath" "sync" "sync/atomic" - "syscall" "time" "github.com/google/syzkaller/dashboard/dashapi" @@ -30,6 +28,7 @@ import ( "github.com/google/syzkaller/pkg/report" "github.com/google/syzkaller/pkg/repro" . "github.com/google/syzkaller/pkg/rpctype" + "github.com/google/syzkaller/pkg/signal" "github.com/google/syzkaller/prog" "github.com/google/syzkaller/sys" "github.com/google/syzkaller/syz-manager/mgrconfig" @@ -72,9 +71,9 @@ type Manager struct { candidates []RpcCandidate // untriaged inputs from corpus and hub disabledHashes map[string]struct{} corpus map[string]RpcInput - corpusSignal map[uint32]struct{} - maxSignal map[uint32]struct{} - corpusCover map[uint32]struct{} + corpusCover cover.Cover + corpusSignal signal.Signal + maxSignal signal.Signal prios [][]float32 newRepros [][]byte @@ -107,7 +106,7 @@ const currentDBVersion = 2 type Fuzzer struct { name string inputs []RpcInput - newMaxSignal []uint32 + newMaxSignal signal.Signal } type Crash struct { @@ -169,9 +168,6 @@ func RunManager(cfg *mgrconfig.Config, target *prog.Target, syscalls map[int]boo enabledSyscalls: enabledSyscalls, corpus: make(map[string]RpcInput), disabledHashes: make(map[string]struct{}), - corpusSignal: make(map[uint32]struct{}), - maxSignal: make(map[uint32]struct{}), - corpusCover: make(map[uint32]struct{}), fuzzers: make(map[string]*Fuzzer), fresh: true, vmStop: make(chan bool), @@ -281,7 +277,7 @@ func RunManager(cfg *mgrconfig.Config, target *prog.Target, syscalls map[int]boo mgr.fuzzingTime += diff * time.Duration(atomic.LoadUint32(&mgr.numFuzzing)) executed := mgr.stats["exec total"] crashes := mgr.stats["crashes"] - signal := len(mgr.corpusSignal) + signal := mgr.corpusSignal.Len() mgr.mu.Unlock() numReproducing := atomic.LoadUint32(&mgr.numReproducing) numFuzzing := atomic.LoadUint32(&mgr.numFuzzing) @@ -309,7 +305,7 @@ func RunManager(cfg *mgrconfig.Config, target *prog.Target, syscalls map[int]boo vals["corpus"] = uint64(len(mgr.corpus)) vals["uptime"] = uint64(time.Since(mgr.firstConnect)) / 1e9 vals["fuzzing"] = uint64(mgr.fuzzingTime) / 1e9 - vals["signal"] = uint64(len(mgr.corpusSignal)) + vals["signal"] = uint64(mgr.corpusSignal.Len()) vals["coverage"] = uint64(len(mgr.corpusCover)) for k, v := range mgr.stats { vals[k] = v @@ -340,16 +336,7 @@ func RunManager(cfg *mgrconfig.Config, target *prog.Target, syscalls map[int]boo }() } - go func() { - c := make(chan os.Signal, 2) - signal.Notify(c, syscall.SIGINT) - <-c - close(vm.Shutdown) - Logf(0, "shutting down...") - <-c - Fatalf("terminating") - }() - + osutil.HandleInterrupts(vm.Shutdown) mgr.vmLoop() } @@ -815,15 +802,17 @@ func (mgr *Manager) getReporter() report.Reporter { func (mgr *Manager) minimizeCorpus() { if mgr.cfg.Cover && len(mgr.corpus) != 0 { - var cov []cover.Cover - var inputs []RpcInput + inputs := make([]signal.SignalContext, 0, len(mgr.corpus)) + for _, inp := range mgr.corpus { - cov = append(cov, inp.Signal) - inputs = append(inputs, inp) + inputs = append(inputs, signal.SignalContext{ + Signal: inp.Signal.Deserialize(), + Context: inp, + }) } newCorpus := make(map[string]RpcInput) - for _, idx := range cover.Minimize(cov) { - inp := inputs[idx] + for _, ctx := range signal.Minimize(inputs) { + inp := ctx.(RpcInput) newCorpus[hash.String(inp.Prog)] = inp } Logf(1, "minimized corpus: %v -> %v", len(mgr.corpus), len(newCorpus)) @@ -883,18 +872,13 @@ func (mgr *Manager) Connect(a *ConnectArgs, r *ConnectRes) error { mgr.prios = prios } - f.inputs = nil for _, inp := range mgr.corpus { r.Inputs = append(r.Inputs, inp) } r.Prios = mgr.prios r.EnabledCalls = mgr.enabledSyscalls r.NeedCheck = !mgr.vmChecked - r.MaxSignal = make([]uint32, 0, len(mgr.maxSignal)) - for s := range mgr.maxSignal { - r.MaxSignal = append(r.MaxSignal, s) - } - f.newMaxSignal = nil + r.MaxSignal = mgr.maxSignal.Serialize() for i := 0; i < mgr.cfg.Procs && len(mgr.candidates) > 0; i++ { last := len(mgr.candidates) - 1 r.Candidates = append(r.Candidates, mgr.candidates[last]) @@ -942,7 +926,9 @@ func (mgr *Manager) Check(a *CheckArgs, r *int) error { } func (mgr *Manager) NewInput(a *NewInputArgs, r *int) error { - Logf(4, "new input from %v for syscall %v (signal=%v cover=%v)", a.Name, a.Call, len(a.Signal), len(a.Cover)) + inputSignal := a.Signal.Deserialize() + Logf(4, "new input from %v for syscall %v (signal=%v, cover=%v)", + a.Name, a.Call, inputSignal.Len(), len(a.Cover)) mgr.mu.Lock() defer mgr.mu.Unlock() @@ -956,17 +942,21 @@ func (mgr *Manager) NewInput(a *NewInputArgs, r *int) error { Logf(0, "failed to deserialize program from fuzzer: %v\n%s", err, a.RpcInput.Prog) return nil } - if !cover.SignalNew(mgr.corpusSignal, a.Signal) { + if mgr.corpusSignal.Diff(inputSignal).Empty() { return nil } mgr.stats["manager new inputs"]++ - cover.SignalAdd(mgr.corpusSignal, a.Signal) - cover.SignalAdd(mgr.corpusCover, a.Cover) + mgr.corpusSignal.Merge(inputSignal) + mgr.corpusCover.Merge(a.Cover) sig := hash.String(a.RpcInput.Prog) if inp, ok := mgr.corpus[sig]; ok { // The input is already present, but possibly with diffent signal/coverage/call. - inp.Signal = cover.Union(inp.Signal, a.RpcInput.Signal) - inp.Cover = cover.Union(inp.Cover, a.RpcInput.Cover) + inputSignal.Merge(inp.Signal.Deserialize()) + inp.Signal = inputSignal.Serialize() + var inputCover cover.Cover + inputCover.Merge(inp.Cover) + inputCover.Merge(a.RpcInput.Cover) + inp.Cover = inputCover.Serialize() mgr.corpus[sig] = inp } else { mgr.corpus[sig] = a.RpcInput @@ -998,22 +988,20 @@ func (mgr *Manager) Poll(a *PollArgs, r *PollRes) error { if f == nil { Fatalf("fuzzer %v is not connected", a.Name) } - var newMaxSignal []uint32 - for _, s := range a.MaxSignal { - if _, ok := mgr.maxSignal[s]; ok { - continue + newMaxSignal := mgr.maxSignal.Diff(a.MaxSignal.Deserialize()) + if !newMaxSignal.Empty() { + mgr.maxSignal.Merge(newMaxSignal) + for _, f1 := range mgr.fuzzers { + if f1 == f { + continue + } + f1.newMaxSignal.Merge(newMaxSignal) } - mgr.maxSignal[s] = struct{}{} - newMaxSignal = append(newMaxSignal, s) } - for _, f1 := range mgr.fuzzers { - if f1 == f { - continue - } - f1.newMaxSignal = append(f1.newMaxSignal, newMaxSignal...) + if !f.newMaxSignal.Empty() { + r.MaxSignal = f.newMaxSignal.Serialize() + f.newMaxSignal = nil } - r.MaxSignal = f.newMaxSignal - f.newMaxSignal = nil for i := 0; i < 100 && len(f.inputs) > 0; i++ { last := len(f.inputs) - 1 r.NewInputs = append(r.NewInputs, f.inputs[last]) @@ -1040,8 +1028,7 @@ func (mgr *Manager) Poll(a *PollArgs, r *PollRes) error { } } } - Logf(4, "poll from %v: recv maxsignal=%v, send maxsignal=%v candidates=%v inputs=%v", - a.Name, len(a.MaxSignal), len(r.MaxSignal), len(r.Candidates), len(r.NewInputs)) + Logf(4, "poll from %v: candidates=%v inputs=%v", a.Name, len(r.Candidates), len(r.NewInputs)) return nil } @@ -1241,7 +1228,7 @@ func (mgr *Manager) dashboardReporter() { Addr: webAddr, UpTime: time.Since(mgr.firstConnect), Corpus: uint64(len(mgr.corpus)), - Cover: uint64(len(mgr.corpusSignal)), + Cover: uint64(mgr.corpusSignal.Len()), FuzzingTime: mgr.fuzzingTime - lastFuzzingTime, Crashes: crashes - lastCrashes, Execs: execs - lastExecs,