reactos/syzkaller
1
0
Fork 0
mirror of https://github.com/reactos/syzkaller.git synced 2024-11-23 11:29:46 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

remove master and naming overhaul

Browse Source 
Remove master process entirely, it is not useful in its current form.
We first need to understand what we want from it, and them re-implement it.

Prefix all binaries with syz- to avoid name clashes.
This commit is contained in:
Dmitry Vyukov 2015-12-17 16:06:33 +01:00
parent 06e6726537
commit 8e7ca7c5ff
15 changed files with 68 additions and 438 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
Makefile
View File

@ -1,24 +1,18 @@
# Copyright 2015 syzkaller project authors. All rights reserved. # Copyright 2015 syzkaller project authors. All rights reserved.
# Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. # Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
.PHONY: all bin format clean master manager fuzzer executor .PHONY: all format clean master manager fuzzer executor
all: master manager fuzzer executor all: manager fuzzer executor
bin: manager:
mkdir -p bin go build -o ./bin/syz-manager github.com/google/syzkaller/syz-manager
master: bin fuzzer:
go build -o ./bin/master github.com/google/syzkaller/master go build -o ./bin/syz-fuzzer github.com/google/syzkaller/syz-fuzzer
manager: bin executor:
go build -o ./bin/manager github.com/google/syzkaller/manager gcc -o ./bin/syz-executor executor/executor.cc -lpthread -static -Wall -O1 -g
fuzzer: bin
go build -o ./bin/fuzzer github.com/google/syzkaller/fuzzer
executor: bin
gcc executor/executor.cc -o ./bin/executor -lpthread -static -Wall -O1 -g
format: format:
go fmt ./... go fmt ./...

95
master/html.go
View File

@ -1,95 +0,0 @@
// Copyright 2015 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
package main
import (
"encoding/json"
"fmt"
"html/template"
"io/ioutil"
"net/http"
)
func (m *Master) httpInfo(w http.ResponseWriter, r *http.Request) {
m.mu.Lock()
defer m.mu.Unlock()
data := &UIData{
CorpusLen: len(m.corpus.m),
}
for _, mgr := range m.managers {
data.Managers = append(data.Managers, UIManager{
Name: mgr.name,
Http: mgr.http,
})
}
if err := htmlTemplate.Execute(w, data); err != nil {
http.Error(w, fmt.Sprintf("failed to execute template: %v", err), http.StatusInternalServerError)
}
}
func (m *Master) httpMinimize(w http.ResponseWriter, r *http.Request) {
corpus := make(map[string]bool)
for _, mgr := range m.managers {
resp, err := http.Get("http://" + mgr.http + "/current_corpus")
if err != nil {
http.Error(w, fmt.Sprintf("failed to query corpus from %v: %v", mgr.name, err), http.StatusInternalServerError)
return
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
http.Error(w, fmt.Sprintf("failed to query corpus from %v: %v", mgr.name, err), http.StatusInternalServerError)
return
}
var hashes []string
err = json.Unmarshal(data, &hashes)
if err != nil || len(hashes) == 0 {
http.Error(w, fmt.Sprintf("failed to parse corpus from %v: %v", mgr.name, err), http.StatusInternalServerError)
return
}
for _, hash := range hashes {
corpus[hash] = true
}
}
m.mu.Lock()
defer m.mu.Unlock()
orig := len(m.corpus.m)
m.corpus.minimize(corpus)
fmt.Printf("minimized: %v -> %v -> %v\n", orig, len(corpus), len(m.corpus.m))
for _, mgr := range m.managers {
mgr.input = 0
}
}
type UIData struct {
CorpusLen int
Managers []UIManager
}
type UIManager struct {
Name string
Http string
}
var htmlTemplate = template.Must(template.New("").Parse(`
<!doctype html>
<html>
<head>
<title>syzkaller master</title>
</head>
<body>
Corpus: {{.CorpusLen}} <br>
{{if .Managers}}
Managers:<br>
{{range $mgr := $.Managers}}
<a href='http://{{$mgr.Http}}'>{{$mgr.Name}}</a><br>
{{end}}
{{else}}
No managers connected<br>
{{end}}
</body></html>
`))

170
master/master.go
View File

@ -1,170 +0,0 @@
// Copyright 2015 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
package main
import (
"flag"
"fmt"
"log"
"net"
"net/http"
"net/rpc"
"path/filepath"
"sync"
"time"
"github.com/google/syzkaller/prog"
. "github.com/google/syzkaller/rpctype"
)
var (
flagWorkdir = flag.String("workdir", "", "dir with persistent artifacts")
flagAddr = flag.String("addr", "", "RPC listen address to connect managers")
flagHTTP = flag.String("http", "", "HTTP server listen address")
flagV = flag.Int("v", 0, "verbosity")
)
// Master manages persistent fuzzer state (input corpus and crashers).
type Master struct {
mu sync.Mutex
managers map[string]*Manager
corpus *PersistentSet
crashers *PersistentSet
startTime time.Time
lastInput time.Time
}
type Manager struct {
name string
http string
input int
}
func main() {
flag.Parse()
if *flagWorkdir == "" {
fatalf("-workdir is not set")
}
if *flagAddr == "" {
fatalf("-addr is not set")
}
if *flagHTTP == "" {
fatalf("-http is not set")
}
ln, err := net.Listen("tcp", *flagAddr)
if err != nil {
fatalf("failed to listen: %v", err)
}
m := &Master{}
m.managers = make(map[string]*Manager)
m.startTime = time.Now()
m.lastInput = time.Now()
logf(0, "loading corpus...")
m.corpus = newPersistentSet(filepath.Join(*flagWorkdir, "corpus"), func(data []byte) bool {
if _, err := prog.Deserialize(data); err != nil {
logf(0, "deleting broken program: %v\n%s", err, data)
return false
}
return true
})
m.crashers = newPersistentSet(filepath.Join(*flagWorkdir, "crashers"), nil)
http.HandleFunc("/", m.httpInfo)
http.HandleFunc("/minimize", m.httpMinimize)
go func() {
logf(0, "serving http on http://%v", *flagHTTP)
panic(http.ListenAndServe(*flagHTTP, nil))
}()
logf(0, "serving rpc on tcp://%v", *flagAddr)
s := rpc.NewServer()
s.Register(m)
go s.Accept(ln)
m.loop()
}
func (m *Master) loop() {
for range time.NewTicker(1 * time.Second).C {
}
}
// Connect attaches new manager to master.
func (m *Master) Connect(a *MasterConnectArgs, r *MasterConnectRes) error {
logf(1, "connect from %v (http://%v)", a.Name, a.Http)
m.mu.Lock()
defer m.mu.Unlock()
mgr := &Manager{
name: a.Name,
http: a.Http,
}
m.managers[a.Name] = mgr
r.Http = *flagHTTP
return nil
}
// NewInput saves new interesting input on master.
func (m *Master) NewInput(a *NewMasterInputArgs, r *int) error {
p, err := prog.Deserialize(a.Prog)
if err != nil {
logf(0, "bogus new input from %v: %v\n%s\n", a.Name, err, a.Prog)
return fmt.Errorf("the program is bogus: %v", err)
}
m.mu.Lock()
defer m.mu.Unlock()
if !m.corpus.add(a.Prog) {
return nil
}
m.lastInput = time.Now()
logf(1, "new input from %v: %s", a.Name, p)
return nil
}
type NewCrasherArgs struct {
Name string
Text []byte
Suppression []byte
Prog []byte
}
// NewCrasher saves new crasher input on master.
func (m *Master) NewCrasher(a *NewCrasherArgs, r *int) error {
logf(0, "new crasher from %v", a.Name)
m.mu.Lock()
defer m.mu.Unlock()
if !m.crashers.add(a.Text) {
return nil // Already have this.
}
return nil
}
func (m *Master) PollInputs(a *MasterPollArgs, r *MasterPollRes) error {
logf(2, "poll from %v", a.Name)
m.mu.Lock()
defer m.mu.Unlock()
mgr := m.managers[a.Name]
if mgr == nil {
return fmt.Errorf("manager is not connected")
}
for i := 0; i < 100 && mgr.input < len(m.corpus.a); i++ {
r.Inputs = append(r.Inputs, m.corpus.a[mgr.input])
mgr.input++
}
return nil
}
func logf(v int, msg string, args ...interface{}) {
if *flagV >= v {
log.Printf(msg, args...)
}
}
func fatalf(msg string, args ...interface{}) {
log.Fatalf(msg, args...)
}

0
fuzzer/fuzzer.go → syz-fuzzer/fuzzer.go
View File

0
manager/cover.go → syz-manager/cover.go
View File

5
manager/example.cfg → syz-manager/example.cfg
View File

@ -1,7 +1,5 @@
{ {
"name": "my-qemu-asan",
"http": "myhost.com:56741", "http": "myhost.com:56741",
"master": "myhost.com:48342",
"workdir": "/syzkaller/manager/workdir", "workdir": "/syzkaller/manager/workdir",
"vmlinux": "/linux/vmlinux", "vmlinux": "/linux/vmlinux",
"type": "qemu", "type": "qemu",
@ -21,5 +19,8 @@
"keyctl", "keyctl",
"add_key", "add_key",
"request_key" "request_key"
],
"suppressions": [
"some known bug"
] ]
} }

55
manager/html.go → syz-manager/html.go
View File

@ -4,8 +4,6 @@
package main package main
import ( import (
"encoding/hex"
"encoding/json"
"fmt" "fmt"
"html/template" "html/template"
"net/http" "net/http"
@ -23,11 +21,8 @@ func (mgr *Manager) initHttp() {
http.HandleFunc("/corpus", mgr.httpCorpus) http.HandleFunc("/corpus", mgr.httpCorpus)
http.HandleFunc("/cover", mgr.httpCover) http.HandleFunc("/cover", mgr.httpCover)
http.HandleFunc("/prio", mgr.httpPrio) http.HandleFunc("/prio", mgr.httpPrio)
http.HandleFunc("/current_corpus", mgr.httpCurrentCorpus) logf(0, "serving http on http://%v", mgr.cfg.Http)
go func() { go http.ListenAndServe(mgr.cfg.Http, nil)
logf(0, "serving http on http://%v", mgr.cfg.Http)
panic(http.ListenAndServe(mgr.cfg.Http, nil))
}()
} }
func (mgr *Manager) httpInfo(w http.ResponseWriter, r *http.Request) { func (mgr *Manager) httpInfo(w http.ResponseWriter, r *http.Request) {
@ -50,12 +45,9 @@ func (mgr *Manager) httpInfo(w http.ResponseWriter, r *http.Request) {
uptime := time.Since(mgr.startTime) uptime := time.Since(mgr.startTime)
data := &UIData{ data := &UIData{
Name: mgr.cfg.Name, CorpusSize: len(mgr.corpus),
MasterHttp: mgr.masterHttp, TriageQueue: len(mgr.candidates),
MasterCorpusSize: len(mgr.masterCorpus), Uptime: fmt.Sprintf("%v", uptime),
CorpusSize: len(mgr.corpus),
TriageQueue: len(mgr.candidates),
Uptime: fmt.Sprintf("%v", uptime),
} }
secs := uint64(uptime) / 1e9 secs := uint64(uptime) / 1e9
@ -164,34 +156,13 @@ func (mgr *Manager) httpPrio(w http.ResponseWriter, r *http.Request) {
} }
} }
func (mgr *Manager) httpCurrentCorpus(w http.ResponseWriter, r *http.Request) {
mgr.mu.Lock()
defer mgr.mu.Unlock()
mgr.minimizeCorpus()
var hashes []string
for _, inp := range mgr.corpus {
hash := hash(inp.Prog)
hashes = append(hashes, hex.EncodeToString(hash[:]))
}
data, err := json.Marshal(&hashes)
if err != nil {
http.Error(w, fmt.Sprintf("failed to marshal corpus: %v", err), http.StatusInternalServerError)
return
}
w.Write(data)
}
type UIData struct { type UIData struct {
Name string CorpusSize int
MasterHttp string TriageQueue int
MasterCorpusSize int CoverSize int
CorpusSize int Uptime string
TriageQueue int Stats []UIStat
CoverSize int Calls []UICallType
Uptime string
Stats []UIStat
Calls []UICallType
} }
type UIStat struct { type UIStat struct {
@ -235,12 +206,10 @@ var htmlTemplate = template.Must(template.New("").Parse(`
<!doctype html> <!doctype html>
<html> <html>
<head> <head>
<title>syzkaller {{.Name}}</title> <title>syzkaller</title>
</head> </head>
<body> <body>
Manager: {{.Name}} <a href='http://{{.MasterHttp}}'>[master]</a> <br>
Uptime: {{.Uptime}}<br> Uptime: {{.Uptime}}<br>
Master corpus: {{.MasterCorpusSize}} <br>
Corpus: {{.CorpusSize}}<br> Corpus: {{.CorpusSize}}<br>
Triage queue len: {{.TriageQueue}}<br> Triage queue len: {{.TriageQueue}}<br>
<a href='/cover'>Cover: {{.CoverSize}}</a> <br> <a href='/cover'>Cover: {{.CoverSize}}</a> <br>

8
manager/main.go → syz-manager/main.go
View File

@ -26,9 +26,7 @@ var (
) )
type Config struct { type Config struct {
Name string
Http string Http string
Master string
Workdir string Workdir string
Vmlinux string Vmlinux string
Type string Type string
@ -110,15 +108,9 @@ func parseConfig() (*Config, map[int]bool) {
if err := json.Unmarshal(data, cfg); err != nil { if err := json.Unmarshal(data, cfg); err != nil {
fatalf("failed to parse config file: %v", err) fatalf("failed to parse config file: %v", err)
} }
if cfg.Name == "" {
fatalf("config param name is empty")
}
if cfg.Http == "" { if cfg.Http == "" {
fatalf("config param http is empty") fatalf("config param http is empty")
} }
if cfg.Master == "" {
fatalf("config param master is empty")
}
if cfg.Workdir == "" { if cfg.Workdir == "" {
fatalf("config param workdir is empty") fatalf("config param workdir is empty")
} }

145
manager/manager.go → syz-manager/manager.go
View File

@ -4,10 +4,10 @@
package main package main
import ( import (
"crypto/sha1"
"fmt" "fmt"
"net" "net"
"net/rpc" "net/rpc"
"path/filepath"
"sync" "sync"
"time" "time"
@ -18,26 +18,17 @@ import (
"github.com/google/syzkaller/vm" "github.com/google/syzkaller/vm"
) )
type Sig [sha1.Size]byte
func hash(data []byte) Sig {
return Sig(sha1.Sum(data))
}
type Manager struct { type Manager struct {
cfg *Config cfg *Config
master *rpc.Client persistentCorpus *PersistentSet
masterHttp string instances []vm.Instance
instances []vm.Instance startTime time.Time
startTime time.Time stats map[string]uint64
stats map[string]uint64
mu sync.Mutex mu sync.Mutex
masterCorpus [][]byte // mirror of master corpus syscalls map[int]bool
masterHashes map[Sig]struct{} // hashes of master corpus
candidates [][]byte // new untriaged inputs from master
syscalls map[int]bool
candidates [][]byte // untriaged inputs
corpus []RpcInput corpus []RpcInput
corpusCover []cover.Cover corpusCover []cover.Cover
prios [][]float32 prios [][]float32
@ -51,31 +42,29 @@ type Fuzzer struct {
} }
func RunManager(cfg *Config, syscalls map[int]bool, instances []vm.Instance) { func RunManager(cfg *Config, syscalls map[int]bool, instances []vm.Instance) {
// Connect to master.
master, err := rpc.Dial("tcp", cfg.Master)
if err != nil {
fatalf("failed to dial mastger: %v", err)
}
a := &MasterConnectArgs{cfg.Name, cfg.Http}
r := &MasterConnectRes{}
if err := master.Call("Master.Connect", a, r); err != nil {
fatalf("failed to connect to master: %v", err)
}
logf(0, "connected to master at %v", cfg.Master)
mgr := &Manager{ mgr := &Manager{
cfg: cfg, cfg: cfg,
master: master, startTime: time.Now(),
masterHttp: r.Http, stats: make(map[string]uint64),
startTime: time.Now(), instances: instances,
stats: make(map[string]uint64), syscalls: syscalls,
instances: instances, corpusCover: make([]cover.Cover, sys.CallCount),
masterHashes: make(map[Sig]struct{}), fuzzers: make(map[string]*Fuzzer),
syscalls: syscalls,
corpusCover: make([]cover.Cover, sys.CallCount),
fuzzers: make(map[string]*Fuzzer),
} }
logf(0, "loading corpus...")
mgr.persistentCorpus = newPersistentSet(filepath.Join(cfg.Workdir, "corpus"), func(data []byte) bool {
if _, err := prog.Deserialize(data); err != nil {
logf(0, "deleting broken program: %v\n%s", err, data)
return false
}
return true
})
for _, prog := range mgr.persistentCorpus.a {
mgr.candidates = append(mgr.candidates, prog)
}
logf(0, "loaded %v programs", len(mgr.persistentCorpus.m))
// Create HTTP server. // Create HTTP server.
mgr.initHttp() mgr.initHttp()
@ -85,64 +74,15 @@ func RunManager(cfg *Config, syscalls map[int]bool, instances []vm.Instance) {
if err != nil { if err != nil {
fatalf("failed to listen on port %v: %v", cfg.Port, err) fatalf("failed to listen on port %v: %v", cfg.Port, err)
} }
logf(0, "serving rpc on tcp://%v", rpcAddr)
s := rpc.NewServer() s := rpc.NewServer()
s.Register(mgr) s.Register(mgr)
go s.Accept(ln) go s.Accept(ln)
logf(0, "serving rpc on tcp://%v", rpcAddr)
mgr.run()
}
func (mgr *Manager) run() {
mgr.pollMaster()
for _, inst := range mgr.instances { for _, inst := range mgr.instances {
go inst.Run() go inst.Run()
} }
pollTicker := time.NewTicker(10 * time.Second).C select {}
for {
select {
case <-pollTicker:
mgr.mu.Lock()
mgr.pollMaster()
mgr.mu.Unlock()
}
}
}
func (mgr *Manager) pollMaster() {
for {
a := &MasterPollArgs{mgr.cfg.Name}
r := &MasterPollRes{}
if err := mgr.master.Call("Master.PollInputs", a, r); err != nil {
fatalf("failed to poll master: %v", err)
}
logf(3, "polling master, got %v inputs", len(r.Inputs))
if len(r.Inputs) == 0 {
break
}
nextProg:
for _, prg := range r.Inputs {
p, err := prog.Deserialize(prg)
if err != nil {
logf(0, "failed to deserialize master program: %v", err)
continue
}
if mgr.syscalls != nil {
for _, c := range p.Calls {
if !mgr.syscalls[c.Meta.ID] {
continue nextProg
}
}
}
sig := hash(prg)
if _, ok := mgr.masterHashes[sig]; ok {
continue
}
mgr.masterHashes[sig] = struct{}{}
mgr.masterCorpus = append(mgr.masterCorpus, prg)
mgr.candidates = append(mgr.candidates, prg)
}
}
} }
func (mgr *Manager) minimizeCorpus() { func (mgr *Manager) minimizeCorpus() {
@ -178,6 +118,16 @@ func (mgr *Manager) minimizeCorpus() {
corpus = append(corpus, p) corpus = append(corpus, p)
} }
mgr.prios = prog.CalculatePriorities(corpus) mgr.prios = prog.CalculatePriorities(corpus)
// Don't minimize persistent corpus until fuzzers have triaged all inputs from it.
if len(mgr.candidates) == 0 {
hashes := make(map[string]bool)
for _, inp := range mgr.corpus {
h := hash(inp.Prog)
hashes[string(h[:])] = true
}
mgr.persistentCorpus.minimize(hashes)
}
} }
func (mgr *Manager) Connect(a *ManagerConnectArgs, r *ManagerConnectRes) error { func (mgr *Manager) Connect(a *ManagerConnectArgs, r *ManagerConnectRes) error {
@ -208,18 +158,7 @@ func (mgr *Manager) NewInput(a *NewManagerInputArgs, r *int) error {
mgr.corpusCover[call] = cover.Union(mgr.corpusCover[call], a.Cover) mgr.corpusCover[call] = cover.Union(mgr.corpusCover[call], a.Cover)
mgr.corpus = append(mgr.corpus, a.RpcInput) mgr.corpus = append(mgr.corpus, a.RpcInput)
mgr.stats["manager new inputs"]++ mgr.stats["manager new inputs"]++
mgr.persistentCorpus.add(a.RpcInput.Prog)
sig := hash(a.Prog)
if _, ok := mgr.masterHashes[sig]; !ok {
mgr.masterHashes[sig] = struct{}{}
mgr.masterCorpus = append(mgr.masterCorpus, a.Prog)
a1 := &NewMasterInputArgs{mgr.cfg.Name, a.Prog}
if err := mgr.master.Call("Master.NewInput", a1, nil); err != nil {
fatalf("call Master.NewInput failed: %v", err)
}
}
return nil return nil
} }

0
master/persistent.go → syz-manager/persistent.go
View File

0
tools/execprog/execprog.go → tools/syz-execprog/execprog.go
View File

0
tools/mutate/mutate.go → tools/syz-mutate/mutate.go
View File

0
tools/prog2c/prog2c.go → tools/syz-prog2c/prog2c.go
View File

0
tools/stress/stress.go → tools/syz-stress/stress.go
View File

6
vm/qemu/qemu.go
View File

@ -258,8 +258,8 @@ func (inst *Instance) Run() {
inst.Logf("started vm") inst.Logf("started vm")
// Copy the binaries into the instance. // Copy the binaries into the instance.
if !inst.CreateSCPCommand(inst.Fuzzer, "/syzkaller_fuzzer").Wait(1*time.Minute) || if !inst.CreateSCPCommand(inst.Fuzzer, "/syz-fuzzer").Wait(1*time.Minute) ||
!inst.CreateSCPCommand(inst.Executor, "/syzkaller_executor").Wait(1*time.Minute) { !inst.CreateSCPCommand(inst.Executor, "/syz-executor").Wait(1*time.Minute) {
outputMu.Lock() outputMu.Lock()
output = append(output, "\nfailed to scp binaries into the instance\n"...) output = append(output, "\nfailed to scp binaries into the instance\n"...)
inst.SaveCrasher(output) inst.SaveCrasher(output)
@ -280,7 +280,7 @@ func (inst *Instance) Run() {
if inst.cfg.NoDropPrivs { if inst.cfg.NoDropPrivs {
dropprivs = "-dropprivs=0" dropprivs = "-dropprivs=0"
} }
cmd := inst.CreateSSHCommand(fmt.Sprintf("/syzkaller_fuzzer -name %v -executor /syzkaller_executor -manager %v:%v -procs %v -leak=%v %v %v %v", cmd := inst.CreateSSHCommand(fmt.Sprintf("/syz-fuzzer -name %v -executor /syz-executor -manager %v:%v -procs %v -leak=%v %v %v %v",
inst.name, hostAddr, inst.cfg.ManagerPort, inst.cfg.Procs, inst.cfg.Leak, cover, dropprivs, inst.callsFlag)) inst.name, hostAddr, inst.cfg.ManagerPort, inst.cfg.Procs, inst.cfg.Leak, cover, dropprivs, inst.callsFlag))
deadline := start.Add(time.Hour) deadline := start.Add(time.Hour)