add various statistics to http interface

fuzzer/fuzzer.go

@@ -69,6 +69,13 @@ var (
 
 	workerIn  = make(chan *prog.Prog, 10)
 	workerOut = make(chan []Input, 10)
+
+	statExecGen       uint64
+	statExecFuzz      uint64
+	statExecCandidate uint64
+	statExecTriage    uint64
+	statExecMinimize  uint64
+	statNewInput      uint64
 )
 
 func main() {
@@ -136,7 +143,26 @@ func main() {
 			continue
 		}
 		if time.Since(lastPoll) > 10*time.Second {
-			a := &ManagerPollArgs{*flagName}
+			a := &ManagerPollArgs{
+				Name:  *flagName,
+				Stats: make(map[string]uint64),
+			}
+			a.Stats["exec total"] = env.StatExecs
+			env.StatExecs = 0
+			a.Stats["executor restarts"] = env.StatRestarts
+			env.StatRestarts = 0
+			a.Stats["exec gen"] = statExecGen
+			statExecGen = 0
+			a.Stats["exec fuzz"] = statExecFuzz
+			statExecFuzz = 0
+			a.Stats["exec candidate"] = statExecCandidate
+			statExecCandidate = 0
+			a.Stats["exec triage"] = statExecTriage
+			statExecTriage = 0
+			a.Stats["exec minimize"] = statExecMinimize
+			statExecMinimize = 0
+			a.Stats["fuzzer new inputs"] = statNewInput
+			statNewInput = 0
 			r := &ManagerPollRes{}
 			if err := manager.Call("Manager.Poll", a, r); err != nil {
 				panic(err)
@@ -153,7 +179,7 @@ func main() {
 					inp := Input{p, 0, nil}
 					corpus = append(corpus, inp)
 				} else {
-					execute(env, p)
+					execute(env, p, &statExecCandidate)
 				}
 			}
 			if len(r.NewInputs) == 0 && len(r.Candidates) == 0 {
@@ -164,16 +190,16 @@ func main() {
 		if len(corpus) == 0 || i%10 == 0 {
 			p := prog.Generate(rnd, programLength, ct)
 			logf(1, "#%v: generated: %s", i, p)
-			execute(env, p)
+			execute(env, p, &statExecGen)
 			p.Mutate(rnd, programLength, ct)
 			logf(1, "#%v: mutated: %s", i, p)
-			execute(env, p)
+			execute(env, p, &statExecFuzz)
 		} else {
 			inp := corpus[rnd.Intn(len(corpus))]
 			p := inp.p.Clone()
 			p.Mutate(rs, programLength, ct)
 			logf(1, "#%v: mutated: %s <- %s", i, p, inp.p)
-			execute(env, p)
+			execute(env, p, &statExecFuzz)
 		}
 	}
 }
@@ -224,7 +250,7 @@ func triageInput(env *ipc.Env, inp Input) {
 
 	minCover := inp.cover
 	for i := 0; i < 3; i++ {
-		allCover := execute1(env, inp.p)
+		allCover := execute1(env, inp.p, &statExecTriage)
 		if len(allCover[inp.call]) == 0 {
 			// The call was not executed. Happens sometimes, reason unknown.
 			continue
@@ -241,7 +267,7 @@ func triageInput(env *ipc.Env, inp Input) {
 		return
 	}
 	inp.p, inp.call = prog.Minimize(inp.p, inp.call, func(p1 *prog.Prog, call1 int) bool {
-		allCover := execute1(env, p1)
+		allCover := execute1(env, p1, &statExecMinimize)
 		if len(allCover[call1]) == 0 {
 			return false // The call was not executed.
 		}
@@ -260,14 +286,15 @@ func triageInput(env *ipc.Env, inp Input) {
 
 	logf(2, "added new input for %v to corpus:\n%s", call.CallName, data)
 
+	statNewInput++
 	a := &NewManagerInputArgs{*flagName, RpcInput{call.CallName, inp.p.Serialize(), inp.call, []uint32(inp.cover)}}
 	if err := manager.Call("Manager.NewInput", a, nil); err != nil {
 		panic(err)
 	}
 }
 
-func execute(env *ipc.Env, p *prog.Prog) {
-	allCover := execute1(env, p)
+func execute(env *ipc.Env, p *prog.Prog, stat *uint64) {
+	allCover := execute1(env, p, stat)
 	for i, cov := range allCover {
 		if len(cov) == 0 {
 			continue
@@ -283,7 +310,7 @@ func execute(env *ipc.Env, p *prog.Prog) {
 
 var logMu sync.Mutex
 
-func execute1(env *ipc.Env, p *prog.Prog) []cover.Cover {
+func execute1(env *ipc.Env, p *prog.Prog, stat *uint64) []cover.Cover {
 	if *flagSaveProg {
 		f, err := os.Create(fmt.Sprintf("%v.prog", *flagName))
 		if err == nil {
@@ -300,6 +327,7 @@ func execute1(env *ipc.Env, p *prog.Prog) []cover.Cover {
 
 	try := 0
 retry:
+	*stat++
 	output, strace, rawCover, failed, hanged, err := env.Exec(p)
 	if err != nil {
 		if try > 10 {

ipc/ipc.go

@@ -28,6 +28,9 @@ type Env struct {
 	bin     []string
 	timeout time.Duration
 	flags   uint64
+
+	StatExecs    uint64
+	StatRestarts uint64
 }
 
 const (
@@ -124,7 +127,9 @@ func (env *Env) Exec(p *prog.Prog) (output, strace []byte, cov [][]uint32, faile
 		}
 	}
 
+	env.StatExecs++
 	if env.cmd == nil {
+		env.StatRestarts++
 		env.cmd, err0 = makeCommand(env.bin, env.timeout, env.flags, env.inFile, env.outFile)
 		if err0 != nil {
 			return
@@ -188,151 +193,6 @@ func (env *Env) Exec(p *prog.Prog) (output, strace []byte, cov [][]uint32, faile
 	return
 }
 
-/*
-func (env *Env) execBin() (output, strace []byte, failed, hanged bool, err0 error) {
-	dir, err := ioutil.TempDir("./", "syzkaller-testdir")
-	if err != nil {
-		err0 = fmt.Errorf("failed to create temp dir: %v", err)
-		return
-	}
-
-	// Output capture pipe.
-	defer os.RemoveAll(dir)
-	rp, wp, err := os.Pipe()
-	if err != nil {
-		err0 = fmt.Errorf("failed to create pipe: %v", err)
-		return
-	}
-	defer rp.Close()
-	defer wp.Close()
-
-	// Input command pipe.
-	inrp, inwp, err := os.Pipe()
-	if err != nil {
-		err0 = fmt.Errorf("failed to create pipe: %v", err)
-		return
-	}
-	defer inrp.Close()
-	defer inwp.Close()
-
-	// Output command pipe.
-	outrp, outwp, err := os.Pipe()
-	if err != nil {
-		err0 = fmt.Errorf("failed to create pipe: %v", err)
-		return
-	}
-	defer outrp.Close()
-	defer outwp.Close()
-
-	cmd := exec.Command(env.bin[0], env.bin[1:]...)
-	traceFile := ""
-	if env.flags&FlagStrace != 0 {
-		f, err := ioutil.TempFile("./", "syzkaller-strace")
-		if err != nil {
-			err0 = fmt.Errorf("failed to create temp file: %v", err)
-			return
-		}
-		f.Close()
-		defer os.Remove(f.Name())
-		traceFile, _ = filepath.Abs(f.Name())
-		args := []string{"-s", "8", "-o", traceFile}
-		args = append(args, env.bin...)
-		if env.flags&FlagThreaded != 0 {
-			args = append([]string{"-f"}, args...)
-		}
-		cmd = exec.Command("strace", args...)
-	}
-	cmd.ExtraFiles = append(cmd.ExtraFiles, env.inFile, env.outFile, outrp, inwp)
-	cmd.Env = []string{}
-	cmd.Dir = dir
-	if env.flags&FlagDebug == 0 {
-		cmd.Stdout = wp
-		cmd.Stderr = wp
-	} else {
-		cmd.Stdout = os.Stdout
-		cmd.Stderr = os.Stdout
-	}
-	if syscall.Getuid() == 0 {
-		// Running under root, more isolation is possible.
-		cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true, Cloneflags: syscall.CLONE_NEWNS}
-	} else {
-		cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
-	}
-	if _, err := outwp.Write([]byte{0}); err != nil {
-		err0 = fmt.Errorf("failed to write control pipe: %v", err)
-		return
-	}
-	if err := cmd.Start(); err != nil {
-		err0 = fmt.Errorf("failed to start executor binary: %v", err)
-		return
-	}
-	wp.Close()
-	outrp.Close()
-	inwp.Close()
-	done := make(chan bool)
-	hang := make(chan bool)
-	go func() {
-		t := time.NewTimer(env.timeout)
-		select {
-		case <-t.C:
-			// We started the process in its own process group and now kill the whole group.
-			// This solves a potential problem with strace:
-			// if we kill just strace, executor still runs and ReadAll below hangs.
-			fmt.Printf("KILLING %v\n", cmd.Process.Pid)
-			syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
-			syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
-			syscall.Kill(cmd.Process.Pid, syscall.SIGKILL)
-			syscall.Kill(cmd.Process.Pid, syscall.SIGKILL)
-			hang <- true
-		case <-done:
-			t.Stop()
-			hang <- false
-		}
-	}()
-	var tmp [1]byte
-	if n, err := inrp.Read(tmp[:]); n != 1 || err != nil {
-		err0 = fmt.Errorf("failed to read control pipe: %v", err)
-		return
-	}
-	output, err = ioutil.ReadAll(rp)
-	readErr := err
-	close(done)
-	if err = cmd.Wait(); <-hang && err != nil {
-		hanged = true
-		failed = true
-	}
-	if err != nil {
-		output = append(output, []byte(err.Error())...)
-		output = append(output, '\n')
-	}
-	if cmd.ProcessState != nil {
-		sys := cmd.ProcessState.Sys()
-		if ws, ok := sys.(syscall.WaitStatus); ok {
-			// Magic values returned by executor.
-			if ws.ExitStatus() == 67 {
-				err0 = fmt.Errorf("executor failed: %s", output)
-				return
-			}
-			if ws.ExitStatus() == 68 {
-				failed = true
-			}
-		}
-	}
-	if readErr != nil {
-		err0 = fmt.Errorf("failed to read executor output: %v", err)
-		return
-	}
-	if traceFile != "" {
-		strace, err = ioutil.ReadFile(traceFile)
-		if err != nil {
-			err0 = fmt.Errorf("failed to read strace output: %v", err)
-			return
-		}
-	}
-	return
-}
-*/
-
 func createMapping(size int) (f *os.File, mem []byte, err error) {
 	f, err = ioutil.TempFile("./", "syzkaller-shm")
 	if err != nil {

manager/html.go

@@ -11,6 +11,7 @@ import (
 	"net/http"
 	"sort"
 	"strconv"
+	"time"
 
 	"github.com/google/syzkaller/cover"
 	"github.com/google/syzkaller/prog"
@@ -47,14 +48,31 @@ func (mgr *Manager) httpInfo(w http.ResponseWriter, r *http.Request) {
 		cc.cov = cover.Union(cc.cov, cover.Cover(inp.Cover))
 	}
 
+	uptime := time.Since(mgr.startTime)
 	data := &UIData{
 		Name:             mgr.cfg.Name,
 		MasterHttp:       mgr.masterHttp,
 		MasterCorpusSize: len(mgr.masterCorpus),
 		CorpusSize:       len(mgr.corpus),
 		TriageQueue:      len(mgr.candidates),
+		Uptime:           fmt.Sprintf("%v", uptime),
 	}
 
+	secs := uint64(uptime) / 1e9
+	for k, v := range mgr.stats {
+		val := ""
+		if x := v / secs; x >= 10 {
+			val = fmt.Sprintf("%v/sec", x)
+		} else if x := v * 60 / secs; x >= 10 {
+			val = fmt.Sprintf("%v/min", x)
+		} else {
+			x := v * 60 * 60 / secs
+			val = fmt.Sprintf("%v/hour", x)
+		}
+		data.Stats = append(data.Stats, UIStat{Name: k, Value: val})
+	}
+	sort.Sort(UIStatArray(data.Stats))
+
 	var cov cover.Cover
 	for c, cc := range calls {
 		cov = cover.Union(cov, cc.cov)
@@ -171,9 +189,16 @@ type UIData struct {
 	CorpusSize       int
 	TriageQueue      int
 	CoverSize        int
+	Uptime           string
+	Stats            []UIStat
 	Calls            []UICallType
 }
 
+type UIStat struct {
+	Name  string
+	Value string
+}
+
 type UICallType struct {
 	Name   string
 	Inputs int
@@ -200,6 +225,12 @@ func (a UIInputArray) Len() int           { return len(a) }
 func (a UIInputArray) Less(i, j int) bool { return a[i].Cover > a[j].Cover }
 func (a UIInputArray) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
 
+type UIStatArray []UIStat
+
+func (a UIStatArray) Len() int           { return len(a) }
+func (a UIStatArray) Less(i, j int) bool { return a[i].Name < a[j].Name }
+func (a UIStatArray) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+
 var htmlTemplate = template.Must(template.New("").Parse(`
 <!doctype html>
 <html>
@@ -208,11 +239,17 @@ var htmlTemplate = template.Must(template.New("").Parse(`
 </head>
 <body>
 Manager: {{.Name}} <a href='http://{{.MasterHttp}}'>[master]</a> <br>
+Uptime: {{.Uptime}}<br>
 Master corpus: {{.MasterCorpusSize}} <br>
 Corpus: {{.CorpusSize}}<br>
 Triage queue len: {{.TriageQueue}}<br>
 <a href='/cover'>Cover: {{.CoverSize}}</a> <br>
 <br>
+Stats: <br>
+{{range $stat := $.Stats}}
+	{{$stat.Name}}: {{$stat.Value}}<br>
+{{end}}
+<br>
 {{range $c := $.Calls}}
 	{{$c.Name}} <a href='/corpus?call={{$c.Name}}'>inputs:{{$c.Inputs}}</a> <a href='/cover?call={{$c.Name}}'>cover:{{$c.Cover}}</a> <a href='/prio?call={{$c.Name}}'>prio</a> <br>
 {{end}}

manager/manager.go

@@ -29,6 +29,8 @@ type Manager struct {
 	master     *rpc.Client
 	masterHttp string
 	instances  []vm.Instance
+	startTime  time.Time
+	stats      map[string]uint64
 
 	mu           sync.Mutex
 	masterCorpus [][]byte         // mirror of master corpus
@@ -65,6 +67,8 @@ func RunManager(cfg *Config, syscalls map[int]bool, instances []vm.Instance) {
 		cfg:          cfg,
 		master:       master,
 		masterHttp:   r.Http,
+		startTime:    time.Now(),
+		stats:        make(map[string]uint64),
 		instances:    instances,
 		masterHashes: make(map[Sig]struct{}),
 		syscalls:     syscalls,
@@ -181,6 +185,7 @@ func (mgr *Manager) Connect(a *ManagerConnectArgs, r *ManagerConnectRes) error {
 	mgr.mu.Lock()
 	defer mgr.mu.Unlock()
 
+	mgr.stats["vm restarts"]++
 	mgr.minimizeCorpus()
 	mgr.fuzzers[a.Name] = &Fuzzer{
 		name:  a.Name,
@@ -202,6 +207,7 @@ func (mgr *Manager) NewInput(a *NewManagerInputArgs, r *int) error {
 	}
 	mgr.corpusCover[call] = cover.Union(mgr.corpusCover[call], a.Cover)
 	mgr.corpus = append(mgr.corpus, a.RpcInput)
+	mgr.stats["manager new inputs"]++
 
 	sig := hash(a.Prog)
 	if _, ok := mgr.masterHashes[sig]; !ok {
@@ -222,6 +228,10 @@ func (mgr *Manager) Poll(a *ManagerPollArgs, r *ManagerPollRes) error {
 	mgr.mu.Lock()
 	defer mgr.mu.Unlock()
 
+	for k, v := range a.Stats {
+		mgr.stats[k] += v
+	}
+
 	f := mgr.fuzzers[a.Name]
 	if f == nil {
 		fatalf("fuzzer %v is not connected", a.Name)

rpctype/rpctype.go

@@ -48,7 +48,8 @@ type NewManagerInputArgs struct {
 }
 
 type ManagerPollArgs struct {
-	Name string
+	Name  string
+	Stats map[string]uint64
 }
 
 type ManagerPollRes struct {