reactos/syzkaller
1
0
Fork 0
mirror of https://github.com/reactos/syzkaller.git synced 2024-10-08 01:23:20 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #241 from dvyukov/dvyukov-ci

Browse Source 
syz-ci: add continuous integration system
This commit is contained in:
Dmitry Vyukov 2017-06-22 16:27:37 +02:00 committed by GitHub
commit 0b4cf413ea
14 changed files with 1055 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Makefile
View File

@ -6,7 +6,7 @@ ifeq ($(NOSTATIC), 0)
STATIC_FLAG=-static
endif
.PHONY: all format tidy clean manager fuzzer executor execprog mutate prog2c stress extract generate repro db bin/syz-extract bin/syz-sysgen
.PHONY: all format tidy clean manager fuzzer executor execprog ci mutate prog2c stress extract generate repro db bin/syz-extract bin/syz-sysgen
all:
go install ./syz-manager ./syz-fuzzer
@ -36,6 +36,9 @@ fuzzer:
execprog:
go build $(GOFLAGS) -o ./bin/syz-execprog github.com/google/syzkaller/tools/syz-execprog
ci:
go build $(GOFLAGS) -o ./bin/syz-ci github.com/google/syzkaller/syz-ci
repro:
go build $(GOFLAGS) -o ./bin/syz-repro github.com/google/syzkaller/tools/syz-repro

9
dashboard/dash.go
View File

@ -41,6 +41,15 @@ type Patch struct {
Diff []byte
}
func New(client, addr, key string) (*Dashboard, error) {
dash := &Dashboard{
Client: client,
Addr: addr,
Key: key,
}
return dash, nil
}
func (dash *Dashboard) ReportCrash(crash *Crash) error {
return dash.query("add_crash", crash, nil)
}

5
docs/ci.md Normal file
View File

@ -0,0 +1,5 @@
# Continuous integration fuzzing
(syz-ci)[syz-ci/] command provides support for continuous fuzzing with syzkaller.
It runs several syz-manager's, polls and rebuilds images for managers and polls
and rebuilds syzkaller binaries.

9
pkg/fileutil/fileutil.go
View File

@ -14,7 +14,7 @@ import (
"unsafe"
)
// CopyFile copies oldFile to newFile.
// CopyFile atomically copies oldFile to newFile preserving permissions and modification time.
func CopyFile(oldFile, newFile string) error {
oldf, err := os.Open(oldFile)
if err != nil {
@ -25,7 +25,8 @@ func CopyFile(oldFile, newFile string) error {
if err != nil {
return err
}
newf, err := os.Create(newFile)
tmpFile := newFile + ".tmp"
newf, err := os.OpenFile(tmpFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, stat.Mode()&os.ModePerm)
if err != nil {
return err
}
@ -37,10 +38,10 @@ func CopyFile(oldFile, newFile string) error {
if err := newf.Close(); err != nil {
return err
}
if err := os.Chtimes(newFile, stat.ModTime(), stat.ModTime()); err != nil {
if err := os.Chtimes(tmpFile, stat.ModTime(), stat.ModTime()); err != nil {
return err
}
return nil
return os.Rename(tmpFile, newFile)
}
// WriteTempFile writes data to a temp file and returns its name.

13
pkg/git/git.go
View File

@ -22,16 +22,27 @@ func Poll(dir, repo, branch string) (string, error) {
osutil.RunCmd(timeout, dir, "git", "reset", "--hard")
origin, err := osutil.RunCmd(timeout, dir, "git", "remote", "get-url", "origin")
if err != nil || strings.TrimSpace(string(origin)) != repo {
// The repo is here, but it has wrong origin (e.g. repo in config has changed), re-clone.
if err := clone(dir, repo, branch); err != nil {
return "", err
}
}
// Use origin/branch for the case the branch was force-pushed,
// in such case branch is not the same is origin/branch and we will
// stuck with the local version forever (git checkout won't fail).
if _, err := osutil.RunCmd(timeout, dir, "git", "checkout", "origin/"+branch); err != nil {
// No such branch (e.g. branch in config has changed), re-clone.
if err := clone(dir, repo, branch); err != nil {
return "", err
}
}
if _, err := osutil.RunCmd(timeout, dir, "git", "fetch", "--no-tags", "--depth", "1"); err != nil {
// Something else is wrong, re-clone.
if err := clone(dir, repo, branch); err != nil {
return "", err
}
}
if _, err := osutil.RunCmd(timeout, dir, "git", "checkout", branch); err != nil {
if _, err := osutil.RunCmd(timeout, dir, "git", "checkout", "origin/"+branch); err != nil {
return "", err
}
return HeadCommit(dir)

54
pkg/kernel/kernel.go
View File

@ -25,35 +25,43 @@ import (
"github.com/google/syzkaller/pkg/osutil"
)
func Build(dir, compiler, config string, fullConfig bool) error {
const timeout = 10 * time.Minute // default timeout for command invocations
if fullConfig {
if err := ioutil.WriteFile(filepath.Join(dir, ".config"), []byte(config), 0600); err != nil {
return fmt.Errorf("failed to write config file: %v", err)
}
} else {
os.Remove(filepath.Join(dir, ".config"))
configFile := filepath.Join(dir, "syz.config")
if err := ioutil.WriteFile(configFile, []byte(config), 0600); err != nil {
return fmt.Errorf("failed to write config file: %v", err)
}
defer os.Remove(configFile)
if _, err := osutil.RunCmd(timeout, dir, "make", "defconfig"); err != nil {
return err
}
if _, err := osutil.RunCmd(timeout, dir, "make", "kvmconfig"); err != nil {
return err
}
if _, err := osutil.RunCmd(timeout, dir, "scripts/kconfig/merge_config.sh", "-n", ".config", configFile); err != nil {
return err
}
func Build(dir, compiler, config string) error {
if err := fileutil.CopyFile(config, filepath.Join(dir, ".config")); err != nil {
return fmt.Errorf("failed to write config file: %v", err)
}
return build(dir, compiler)
}
// TODO(dvyukov): this is only for syz-gce, remove when syz-gce is deleted.
func BuildWithPartConfig(dir, compiler, config string) error {
const timeout = 10 * time.Minute // default timeout for command invocations
os.Remove(filepath.Join(dir, ".config"))
configFile := filepath.Join(dir, "syz.config")
if err := ioutil.WriteFile(configFile, []byte(config), 0600); err != nil {
return fmt.Errorf("failed to write config file: %v", err)
}
defer os.Remove(configFile)
if _, err := osutil.RunCmd(timeout, dir, "make", "defconfig"); err != nil {
return err
}
if _, err := osutil.RunCmd(timeout, dir, "make", "kvmconfig"); err != nil {
return err
}
if _, err := osutil.RunCmd(timeout, dir, "scripts/kconfig/merge_config.sh", "-n", ".config", configFile); err != nil {
return err
}
return build(dir, compiler)
}
func build(dir, compiler string) error {
const timeout = 10 * time.Minute // default timeout for command invocations
if _, err := osutil.RunCmd(timeout, dir, "make", "olddefconfig"); err != nil {
return err
}
// We build only bzImage as we currently don't use modules.
// Build of a large kernel can take a while on a 1 CPU VM.
if _, err := osutil.RunCmd(3*time.Hour, dir, "make", "bzImage", "-j", strconv.Itoa(runtime.NumCPU()), "CC="+compiler); err != nil {
cpu := strconv.Itoa(runtime.NumCPU())
if _, err := osutil.RunCmd(3*time.Hour, dir, "make", "bzImage", "-j", cpu, "CC="+compiler); err != nil {
return err
}
return nil

70
pkg/osutil/osutil.go
View File

@ -13,6 +13,13 @@ import (
"path/filepath"
"syscall"
"time"
"github.com/google/syzkaller/pkg/fileutil"
)
const (
DefaultDirPerm = 0755
DefaultFilePerm = 0644
)
// RunCmd runs "bin args..." in dir with timeout and returns its output.
@ -94,3 +101,66 @@ func HandleInterrupts(shutdown chan struct{}) {
os.Exit(int(syscall.SIGINT))
}()
}
// FilesExist returns true if all files exist in dir.
// Files are assumed to be relative names in slash notation.
func FilesExist(dir string, files []string) bool {
for _, f := range files {
if !IsExist(filepath.Join(dir, filepath.FromSlash(f))) {
return false
}
}
return true
}
// CopyFiles copies files from srcDir to dstDir as atomically as possible.
// Files are assumed to be relative names in slash notation.
// All other files in dstDir are removed.
func CopyFiles(srcDir, dstDir string, files []string) error {
// Linux does not support atomic dir replace, so we copy to tmp dir first.
// Then remove dst dir and rename tmp to dst (as atomic as can get on Linux).
tmpDir := dstDir + ".tmp"
if err := os.RemoveAll(tmpDir); err != nil {
return err
}
if err := os.MkdirAll(tmpDir, DefaultDirPerm); err != nil {
return err
}
for _, f := range files {
src := filepath.Join(srcDir, filepath.FromSlash(f))
dst := filepath.Join(tmpDir, filepath.FromSlash(f))
if err := os.MkdirAll(filepath.Dir(dst), DefaultDirPerm); err != nil {
return err
}
if err := fileutil.CopyFile(src, dst); err != nil {
return err
}
}
if err := os.RemoveAll(dstDir); err != nil {
return err
}
return os.Rename(tmpDir, dstDir)
}
// LinkFiles creates hard links for files from dstDir to srcDir.
// Files are assumed to be relative names in slash notation.
// All other files in dstDir are removed.
func LinkFiles(srcDir, dstDir string, files []string) error {
if err := os.RemoveAll(dstDir); err != nil {
return err
}
if err := os.MkdirAll(dstDir, DefaultDirPerm); err != nil {
return err
}
for _, f := range files {
src := filepath.Join(srcDir, filepath.FromSlash(f))
dst := filepath.Join(dstDir, filepath.FromSlash(f))
if err := os.MkdirAll(filepath.Dir(dst), DefaultDirPerm); err != nil {
return err
}
if err := os.Link(src, dst); err != nil {
return err
}
}
return nil
}

14
syz-ci/config_test.go Normal file
View File

@ -0,0 +1,14 @@
// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
package main
import (
"testing"
)
func TestLoadConfig(t *testing.T) {
if _, err := loadConfig("testdata/example.cfg"); err != nil {
t.Fatalf("failed to load: %v", err)
}
}

324
syz-ci/manager.go Normal file
View File

@ -0,0 +1,324 @@
// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
package main
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"time"
"github.com/google/syzkaller/dashboard"
"github.com/google/syzkaller/pkg/config"
"github.com/google/syzkaller/pkg/fileutil"
"github.com/google/syzkaller/pkg/git"
"github.com/google/syzkaller/pkg/hash"
"github.com/google/syzkaller/pkg/kernel"
. "github.com/google/syzkaller/pkg/log"
"github.com/google/syzkaller/pkg/osutil"
"github.com/google/syzkaller/syz-manager/mgrconfig"
)
// This is especially slightly longer than syzkaller rebuild period.
// If we set kernelRebuildPeriod = syzkallerRebuildPeriod and both are changed
// during that period (or around that period), we can rebuild kernel, restart
// manager and then instantly shutdown everything for syzkaller update.
// Instead we rebuild syzkaller, restart and then rebuild kernel.
const kernelRebuildPeriod = syzkallerRebuildPeriod + time.Hour
// List of required files in kernel build (contents of latest/current dirs).
var imageFiles = []string{
"kernel.tag", // git hash of kernel checkout
"compiler.tag", // compiler identity string (e.g. "gcc 7.1.1")
"kernel.config", // kernel config used for the build (identified with SHA1 hash of contents)
"tag", // SHA1 hash of the previous 3 tags (this is what uniquely identifies the build)
"image", // kernel image
"key", // root ssh key for the image
"obj/vmlinux", // vmlinux with debug info
}
// Manager represents a single syz-manager instance.
// Handles kernel polling, image rebuild and manager process management.
// As syzkaller builder, it maintains 2 builds:
// - latest: latest known good kernel build
// - current: currently used kernel build
type Manager struct {
name string
workDir string
kernelDir string
currentDir string
latestDir string
compilerTag string
configTag string
cfg *Config
mgrcfg *ManagerConfig
cmd *ManagerCmd
dash *dashboard.Dashboard
stop chan struct{}
}
func createManager(dash *dashboard.Dashboard, cfg *Config, mgrcfg *ManagerConfig, stop chan struct{}) *Manager {
dir := osutil.Abs(filepath.Join("managers", mgrcfg.Name))
if err := os.MkdirAll(dir, osutil.DefaultDirPerm); err != nil {
Fatal(err)
}
// Assume compiler and config don't change underneath us.
compilerTag, err := kernel.CompilerIdentity(mgrcfg.Compiler)
if err != nil {
Fatal(err)
}
configData, err := ioutil.ReadFile(mgrcfg.Kernel_Config)
if err != nil {
Fatal(err)
}
mgr := &Manager{
name: mgrcfg.Name,
workDir: filepath.Join(dir, "workdir"),
kernelDir: filepath.Join(dir, "kernel"),
currentDir: filepath.Join(dir, "current"),
latestDir: filepath.Join(dir, "latest"),
compilerTag: compilerTag,
configTag: hash.String(configData),
cfg: cfg,
mgrcfg: mgrcfg,
dash: dash,
stop: stop,
}
os.RemoveAll(mgr.currentDir)
return mgr
}
// Gates kernel builds.
// Kernel builds take whole machine, so we don't run more than one at a time.
// Also current image build script uses some global resources (/dev/nbd0) and can't run in parallel.
var kernelBuildSem = make(chan struct{}, 1)
func (mgr *Manager) loop() {
lastCommit := ""
nextBuildTime := time.Now()
var managerRestartTime time.Time
latestTime, latestKernelTag, latestCompilerTag, latestConfigTag := mgr.checkLatest()
if time.Since(latestTime) < kernelRebuildPeriod/2 {
// If we have a reasonably fresh build,
// start manager straight away and don't rebuild kernel for a while.
Logf(0, "%v: using latest image built on %v", mgr.name, latestKernelTag)
managerRestartTime = latestTime
nextBuildTime = time.Now().Add(kernelRebuildPeriod)
mgr.restartManager()
} else {
Logf(0, "%v: latest image is on %v", mgr.name, formatTag(latestKernelTag))
}
ticker := time.NewTicker(kernelRebuildPeriod)
defer ticker.Stop()
loop:
for {
if time.Since(nextBuildTime) >= 0 {
rebuildAfter := buildRetryPeriod
commit, err := git.Poll(mgr.kernelDir, mgr.mgrcfg.Repo, mgr.mgrcfg.Branch)
if err != nil {
Logf(0, "%v: failed to poll: %v", mgr.name, err)
} else {
Logf(0, "%v: poll: %v", mgr.name, commit)
if commit != lastCommit &&
(commit != latestKernelTag ||
mgr.compilerTag != latestCompilerTag ||
mgr.configTag != latestConfigTag) {
lastCommit = commit
select {
case kernelBuildSem <- struct{}{}:
Logf(0, "%v: building kernel...", mgr.name)
if err := mgr.build(); err != nil {
Logf(0, "%v: %v", mgr.name, err)
} else {
Logf(0, "%v: build successful, [re]starting manager", mgr.name)
rebuildAfter = kernelRebuildPeriod
latestTime, latestKernelTag, latestCompilerTag, latestConfigTag = mgr.checkLatest()
}
<-kernelBuildSem
case <-mgr.stop:
break loop
}
}
}
nextBuildTime = time.Now().Add(rebuildAfter)
}
select {
case <-mgr.stop:
break loop
default:
}
if managerRestartTime != latestTime {
managerRestartTime = latestTime
mgr.restartManager()
}
select {
case <-ticker.C:
case <-mgr.stop:
break loop
}
}
if mgr.cmd != nil {
mgr.cmd.Close()
mgr.cmd = nil
}
Logf(0, "%v: stopped", mgr.name)
}
// checkLatest checks if we have a good working latest build
// and returns its kernel/compiler/config tags.
// If the build is missing/broken, zero mod time is returned.
func (mgr *Manager) checkLatest() (mod time.Time, kernelTag, compilerTag, configTag string) {
if !osutil.FilesExist(mgr.latestDir, imageFiles) {
return
}
configData, err := ioutil.ReadFile(filepath.Join(mgr.latestDir, "kernel.config"))
if err != nil {
return
}
configTag = hash.String(configData)
compilerTag, _ = readTag(filepath.Join(mgr.latestDir, "compiler.tag"))
if compilerTag == "" {
return
}
kernelTag, mod = readTag(filepath.Join(mgr.latestDir, "kernel.tag"))
return
}
func (mgr *Manager) build() error {
kernelCommit, err := git.HeadCommit(mgr.kernelDir)
if err != nil {
return fmt.Errorf("failed to get git HEAD commit: %v", err)
}
if err := kernel.Build(mgr.kernelDir, mgr.mgrcfg.Compiler, mgr.mgrcfg.Kernel_Config); err != nil {
return fmt.Errorf("kernel build failed: %v", err)
}
// We first form the whole image in tmp dir and then rename it to latest.
tmpDir := mgr.latestDir + ".tmp"
if err := os.RemoveAll(tmpDir); err != nil {
return fmt.Errorf("failed to remove tmp dir: %v", err)
}
if err := os.MkdirAll(tmpDir, osutil.DefaultDirPerm); err != nil {
return fmt.Errorf("failed to create tmp dir: %v", err)
}
image := filepath.Join(tmpDir, "image")
key := filepath.Join(tmpDir, "key")
if err := kernel.CreateImage(mgr.kernelDir, mgr.mgrcfg.Userspace, image, key); err != nil {
return fmt.Errorf("image build failed: %v", err)
}
// TODO(dvyukov): test that the image is good (boots and we can ssh into it).
vmlinux := filepath.Join(mgr.kernelDir, "vmlinux")
objDir := filepath.Join(tmpDir, "obj")
os.MkdirAll(objDir, osutil.DefaultDirPerm)
if err := os.Rename(vmlinux, filepath.Join(objDir, "vmlinux")); err != nil {
return fmt.Errorf("failed to rename vmlinux file: %v", err)
}
kernelConfig := filepath.Join(tmpDir, "kernel.config")
if err := fileutil.CopyFile(mgr.mgrcfg.Kernel_Config, kernelConfig); err != nil {
return err
}
writeTagFile := func(filename, data string) error {
f := filepath.Join(tmpDir, filename)
if err := ioutil.WriteFile(f, []byte(data), osutil.DefaultFilePerm); err != nil {
return fmt.Errorf("failed to write tag file: %v", err)
}
return nil
}
if err := writeTagFile("kernel.tag", kernelCommit); err != nil {
return err
}
if err := writeTagFile("compiler.tag", mgr.compilerTag); err != nil {
return err
}
var tag []byte
tag = append(tag, kernelCommit...)
tag = append(tag, mgr.configTag...)
tag = append(tag, mgr.compilerTag...)
if err := writeTagFile("tag", hash.String(tag)); err != nil {
return err
}
// Now try to replace latest with our tmp dir as atomically as we can get on Linux.
if err := os.RemoveAll(mgr.latestDir); err != nil {
return fmt.Errorf("failed to remove latest dir: %v", err)
}
return os.Rename(tmpDir, mgr.latestDir)
}
func (mgr *Manager) restartManager() {
if !osutil.FilesExist(mgr.latestDir, imageFiles) {
Logf(0, "%v: can't start manager, image files missing", mgr.name)
return
}
if mgr.cmd != nil {
mgr.cmd.Close()
mgr.cmd = nil
}
if err := osutil.LinkFiles(mgr.latestDir, mgr.currentDir, imageFiles); err != nil {
Logf(0, "%v: failed to create current image dir: %v", mgr.name, err)
return
}
cfgFile, err := mgr.writeConfig()
if err != nil {
Logf(0, "%v: failed to create manager config: %v", mgr.name, err)
return
}
bin := filepath.FromSlash("syzkaller/current/bin/syz-manager")
logFile := filepath.Join(mgr.currentDir, "manager.log")
mgr.cmd = NewManagerCmd(mgr.name, logFile, bin, "-config", cfgFile)
}
func (mgr *Manager) writeConfig() (string, error) {
mgrcfg := &mgrconfig.Config{
Cover: true,
Reproduce: true,
Sandbox: "setuid",
Rpc: "localhost:0",
Procs: 1,
}
err := config.LoadData(mgr.mgrcfg.Manager_Config, mgrcfg)
if err != nil {
return "", err
}
current := mgr.currentDir
// TODO(dvyukov): we use kernel.tag because dashboard does not support build info yet.
// Later we should use tag file because it identifies kernel+compiler+config.
tag, err := ioutil.ReadFile(filepath.Join(current, "kernel.tag"))
if err != nil {
return "", fmt.Errorf("failed to read tag file: %v", err)
}
mgrcfg.Name = mgr.cfg.Name + "-" + mgr.name
mgrcfg.Hub_Addr = mgr.cfg.Hub_Addr
mgrcfg.Hub_Key = mgr.cfg.Hub_Key
mgrcfg.Dashboard_Addr = mgr.cfg.Dashboard_Addr
mgrcfg.Dashboard_Key = mgr.cfg.Dashboard_Key
mgrcfg.Workdir = mgr.workDir
mgrcfg.Vmlinux = filepath.Join(current, "obj", "vmlinux")
mgrcfg.Tag = string(tag)
mgrcfg.Syzkaller = filepath.FromSlash("syzkaller/current")
mgrcfg.Image = filepath.Join(current, "image")
mgrcfg.Sshkey = filepath.Join(current, "key")
configFile := filepath.Join(current, "manager.cfg")
if err := config.SaveFile(configFile, mgrcfg); err != nil {
return "", err
}
if _, _, err := mgrconfig.LoadFile(configFile); err != nil {
return "", err
}
return configFile, nil
}

120
syz-ci/managercmd.go Normal file
View File

@ -0,0 +1,120 @@
// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
package main
import (
"os"
"os/exec"
"syscall"
"time"
. "github.com/google/syzkaller/pkg/log"
)
// ManagerCmd encapsulates a single instance of syz-manager process.
// It automatically restarts syz-manager if it exits unexpectedly,
// and supports graceful shutdown via SIGINT.
type ManagerCmd struct {
name string
log string
bin string
args []string
closing chan bool
}
// NewManagerCmd starts new syz-manager process.
// name - name for logging.
// log - manager log file with stdout/stderr.
// bin/args - process binary/args.
func NewManagerCmd(name, log, bin string, args ...string) *ManagerCmd {
mc := &ManagerCmd{
name: name,
log: log,
bin: bin,
args: args,
closing: make(chan bool),
}
go mc.loop()
return mc
}
// Close gracefully shutdowns the process and waits for its termination.
func (mc *ManagerCmd) Close() {
mc.closing <- true
<-mc.closing
}
func (mc *ManagerCmd) loop() {
const (
restartPeriod = time.Minute // don't restart crashing manager more frequently than that
interruptTimeout = time.Minute // give manager that much time to react to SIGINT
)
var (
cmd *exec.Cmd
started time.Time
interrupted time.Time
stopped = make(chan error, 1)
closing = mc.closing
ticker1 = time.NewTicker(restartPeriod)
ticker2 = time.NewTicker(interruptTimeout)
)
defer func() {
ticker1.Stop()
ticker2.Stop()
}()
for closing != nil || cmd != nil {
if cmd == nil {
// cmd is not running
// don't restart too frequently (in case it instantly exits with an error)
if time.Since(started) > restartPeriod {
started = time.Now()
logfile, err := os.Create(mc.log)
if err != nil {
Logf(1, "%v: failed to create manager log: %v", mc.name, err)
} else {
cmd = exec.Command(mc.bin, mc.args...)
cmd.Stdout = logfile
cmd.Stderr = logfile
err := cmd.Start()
logfile.Close()
if err != nil {
Logf(1, "%v: failed to start manager: %v", mc.name, err)
cmd = nil
} else {
Logf(1, "%v: started manager", mc.name)
go func() {
stopped <- cmd.Wait()
}()
}
}
}
} else {
// cmd is running
if closing == nil && time.Since(interrupted) > interruptTimeout {
Logf(1, "%v: killing manager", mc.name)
cmd.Process.Kill()
interrupted = time.Now()
}
}
select {
case <-closing:
closing = nil
if cmd != nil {
Logf(1, "%v: stopping manager", mc.name)
cmd.Process.Signal(syscall.SIGINT)
interrupted = time.Now()
}
case err := <-stopped:
if cmd == nil {
panic("spurious stop signal")
}
cmd = nil
Logf(1, "%v: manager exited with %v", mc.name, err)
case <-ticker1.C:
case <-ticker2.C:
}
}
close(mc.closing)
}

177
syz-ci/syz-ci.go Normal file
View File

@ -0,0 +1,177 @@
// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
// syz-ci is a continuous fuzzing system for syzkaller.
// It runs several syz-manager's, polls and rebuilds images for managers
// and polls and rebuilds syzkaller binaries.
// For usage instructions see: docs/ci.md
package main
// Implementation details:
//
// 2 main components:
// - SyzUpdater: handles syzkaller updates
// - Manager: handles kernel build and syz-manager process (one per manager)
// Both operate in a similar way and keep 2 builds:
// - latest: latest known good build (i.e. we tested it)
// preserved across restarts/reboots, i.e. we can start fuzzing even when
// current syzkaller/kernel git head is broken, or git is down, or anything else
// - current: currently used build (a copy of one of the latest builds)
// Other important points:
// - syz-ci is always built on the same revision as the rest of syzkaller binaries,
// this allows us to handle e.g. changes in manager config format.
// - consequently, syzkaller binaries are never updated on-the-fly,
// instead we re-exec and then update
// - we understand when the latest build is fresh even after reboot,
// i.e. we store enough information to identify it (git hash, compiler identity, etc),
// so we don't rebuild unnecessary (kernel builds take time)
// - we generally avoid crashing the process and handle all errors gracefully
// (this is a continuous system), except for some severe/user errors during start
// (e.g. bad config file, or can't create necessary dirs)
//
// Directory/file structure:
// syz-ci : current executable
// syz-ci.tag : tag of the current executable (syzkaller git hash)
// syzkaller/
// latest/ : latest good syzkaller build
// current/ : syzkaller build currently in use
// managers/
// manager1/ : one dir per manager
// kernel/ : kernel checkout
// workdir/ : manager workdir (never deleted)
// latest/ : latest good kernel image build
// current/ : kernel image currently in use
//
// Current executable, syzkaller and kernel builds are marked with tag files.
// Tag files uniquely identify the build (git hash, compiler identity, kernel config, etc).
// For tag files both contents and modification time are important,
// modification time allows us to understand if we need to rebuild after a restart.
import (
"encoding/json"
"flag"
"fmt"
"os"
"sync"
"github.com/google/syzkaller/dashboard"
"github.com/google/syzkaller/pkg/config"
. "github.com/google/syzkaller/pkg/log"
"github.com/google/syzkaller/pkg/osutil"
"github.com/google/syzkaller/syz-manager/mgrconfig"
)
var flagConfig = flag.String("config", "", "config file")
type Config struct {
Name string
Http string
Dashboard_Addr string
Dashboard_Key string
Hub_Addr string
Hub_Key string
Goroot string
Syzkaller_Repo string
Syzkaller_Branch string
Managers []*ManagerConfig
}
type ManagerConfig struct {
Name string
Repo string
Branch string
Compiler string
Userspace string
Kernel_Config string
Manager_Config json.RawMessage
}
func main() {
flag.Parse()
EnableLogCaching(1000, 1<<20)
cfg, err := loadConfig(*flagConfig)
if err != nil {
Fatalf("failed to load config: %v", err)
}
shutdownPending := make(chan struct{})
osutil.HandleInterrupts(shutdownPending)
updater := NewSyzUpdater(cfg)
updater.UpdateOnStart(shutdownPending)
updatePending := make(chan struct{})
go func() {
updater.WaitForUpdate()
close(updatePending)
}()
stop := make(chan struct{})
go func() {
select {
case <-shutdownPending:
case <-updatePending:
}
close(stop)
}()
var dash *dashboard.Dashboard
if cfg.Dashboard_Addr != "" {
dash, err = dashboard.New(cfg.Name, cfg.Dashboard_Addr, cfg.Dashboard_Key)
if err != nil {
Fatalf("failed to create dashboard client: %v", err)
}
}
var wg sync.WaitGroup
wg.Add(len(cfg.Managers))
managers := make([]*Manager, len(cfg.Managers))
for i, mgrcfg := range cfg.Managers {
managers[i] = createManager(dash, cfg, mgrcfg, stop)
}
for _, mgr := range managers {
mgr := mgr
go func() {
defer wg.Done()
mgr.loop()
}()
}
<-stop
wg.Wait()
select {
case <-shutdownPending:
case <-updatePending:
updater.UpdateAndRestart()
}
}
func loadConfig(filename string) (*Config, error) {
cfg := &Config{
Syzkaller_Repo: "https://github.com/google/syzkaller.git",
Syzkaller_Branch: "master",
Goroot: os.Getenv("GOROOT"),
}
if err := config.LoadFile(filename, cfg); err != nil {
return nil, err
}
if cfg.Name == "" {
return nil, fmt.Errorf("param 'name' is empty")
}
if cfg.Http == "" {
return nil, fmt.Errorf("param 'http' is empty")
}
if len(cfg.Managers) == 0 {
return nil, fmt.Errorf("no managers specified")
}
for i, mgr := range cfg.Managers {
if mgr.Name == "" {
return nil, fmt.Errorf("param 'managers[%v].name' is empty", i)
}
mgrcfg := new(mgrconfig.Config)
if err := config.LoadData(mgr.Manager_Config, mgrcfg); err != nil {
return nil, fmt.Errorf("manager %v: %v", mgr.Name, err)
}
}
return cfg, nil
}

229
syz-ci/syzupdater.go Normal file
View File

@ -0,0 +1,229 @@
// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
package main
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"syscall"
"time"
"github.com/google/syzkaller/pkg/fileutil"
"github.com/google/syzkaller/pkg/git"
. "github.com/google/syzkaller/pkg/log"
"github.com/google/syzkaller/pkg/osutil"
)
const (
syzkallerRebuildPeriod = 12 * time.Hour
buildRetryPeriod = 15 * time.Minute // used for both syzkaller and kernel
)
// List of required files in syzkaller build (contents of latest/current dirs).
var syzFiles = []string{
"tag", // contains syzkaller repo git hash
"bin/syz-ci", // these are just copied from syzkaller dir
"bin/syz-manager",
"bin/syz-fuzzer",
"bin/syz-executor",
"bin/syz-execprog",
}
// SyzUpdater handles everything related to syzkaller updates.
// As kernel builder, it maintains 2 builds:
// - latest: latest known good syzkaller build
// - current: currently used syzkaller build
// Additionally it updates and restarts the current executable as necessary.
// Current executable is always built on the same revision as the rest of syzkaller binaries.
type SyzUpdater struct {
exe string
repo string
branch string
syzkallerDir string
latestDir string
currentDir string
}
func NewSyzUpdater(cfg *Config) *SyzUpdater {
wd, err := os.Getwd()
if err != nil {
Fatalf("failed to get wd: %v", err)
}
bin := os.Args[0]
if !filepath.IsAbs(bin) {
bin = filepath.Join(wd, bin)
}
bin = filepath.Clean(bin)
exe := filepath.Base(bin)
if wd != filepath.Dir(bin) {
Fatalf("%v executable must be in cwd (it will be overwritten on update)", exe)
}
gopath := filepath.Join(wd, "gopath")
os.Setenv("GOPATH", gopath)
os.Setenv("GOROOT", cfg.Goroot)
os.Setenv("PATH", filepath.Join(cfg.Goroot, "bin")+
string(filepath.ListSeparator)+os.Getenv("PATH"))
syzkallerDir := filepath.Join(gopath, "src", "github.com", "google", "syzkaller")
os.MkdirAll(syzkallerDir, osutil.DefaultDirPerm)
return &SyzUpdater{
exe: exe,
repo: cfg.Syzkaller_Repo,
branch: cfg.Syzkaller_Branch,
syzkallerDir: syzkallerDir,
latestDir: filepath.Join("syzkaller", "latest"),
currentDir: filepath.Join("syzkaller", "current"),
}
}
// UpdateOnStart does 3 things:
// - ensures that the current executable is fresh
// - ensures that we have a working syzkaller build in current
func (upd *SyzUpdater) UpdateOnStart(shutdown chan struct{}) {
os.RemoveAll(upd.currentDir)
exeTag, exeMod := readTag(upd.exe + ".tag")
latestTag := upd.checkLatest()
if exeTag == latestTag && time.Since(exeMod) < syzkallerRebuildPeriod/2 {
// Have a freash up-to-date build, probably just restarted.
Logf(0, "current executable is up-to-date (%v)", exeTag)
if err := osutil.LinkFiles(upd.latestDir, upd.currentDir, syzFiles); err != nil {
Fatal(err)
}
return
}
Logf(0, "current executable is on %v", formatTag(exeTag))
Logf(0, "latest syzkaller build is on %v", formatTag(latestTag))
// No syzkaller build or executable is stale.
lastCommit := ""
for {
lastCommit = upd.pollAndBuild(lastCommit)
latestTag := upd.checkLatest()
if latestTag != "" {
// The build was successful or we had the latest build from previous runs.
// Either way, use the latest build.
Logf(0, "using syzkaller built on %v", latestTag)
if err := osutil.LinkFiles(upd.latestDir, upd.currentDir, syzFiles); err != nil {
Fatal(err)
}
if exeTag != latestTag {
upd.UpdateAndRestart()
}
return
}
// No good build at all, try again later.
Logf(0, "retrying in %v", buildRetryPeriod)
select {
case <-time.After(buildRetryPeriod):
case <-shutdown:
os.Exit(0)
}
}
}
// WaitForUpdate polls and rebuilds syzkaller.
// Returns when we have a new good build in latest.
func (upd *SyzUpdater) WaitForUpdate() {
time.Sleep(syzkallerRebuildPeriod)
latestTag := upd.checkLatest()
lastCommit := latestTag
for {
lastCommit = upd.pollAndBuild(lastCommit)
if latestTag != upd.checkLatest() {
break
}
time.Sleep(buildRetryPeriod)
}
Logf(0, "syzkaller: update available, restarting")
}
// UpdateAndRestart updates and restarts the current executable.
// Does not return.
func (upd *SyzUpdater) UpdateAndRestart() {
Logf(0, "restarting executable for update")
latestBin := filepath.Join(upd.latestDir, "bin", upd.exe)
latestTag := filepath.Join(upd.latestDir, "tag")
if err := fileutil.CopyFile(latestBin, upd.exe); err != nil {
Fatal(err)
}
if err := fileutil.CopyFile(latestTag, upd.exe+".tag"); err != nil {
Fatal(err)
}
if err := syscall.Exec(upd.exe, os.Args, os.Environ()); err != nil {
Fatal(err)
}
Fatalf("not reachable")
}
func (upd *SyzUpdater) pollAndBuild(lastCommit string) string {
commit, err := git.Poll(upd.syzkallerDir, upd.repo, upd.branch)
if err != nil {
Logf(0, "syzkaller: failed to poll: %v", err)
} else {
Logf(0, "syzkaller: poll: %v", commit)
if lastCommit != commit {
Logf(0, "syzkaller: building ...")
lastCommit = commit
if err := upd.build(); err != nil {
Logf(0, "syzkaller: %v", err)
}
}
}
return lastCommit
}
func (upd *SyzUpdater) build() error {
commit, err := git.HeadCommit(upd.syzkallerDir)
if err != nil {
return fmt.Errorf("failed to get HEAD commit: %v", err)
}
if _, err := osutil.RunCmd(time.Hour, upd.syzkallerDir, "make", "all", "ci"); err != nil {
return fmt.Errorf("build failed: %v", err)
}
if _, err := osutil.RunCmd(time.Hour, upd.syzkallerDir, "go", "test", "-short", "./..."); err != nil {
return fmt.Errorf("tests failed: %v", err)
}
tagFile := filepath.Join(upd.syzkallerDir, "tag")
if err := ioutil.WriteFile(tagFile, []byte(commit), osutil.DefaultFilePerm); err != nil {
return fmt.Errorf("filed to write tag file: %v", err)
}
if err := osutil.CopyFiles(upd.syzkallerDir, upd.latestDir, syzFiles); err != nil {
return fmt.Errorf("filed to copy syzkaller: %v", err)
}
return nil
}
// checkLatest returns tag of the latest build,
// or an empty string if latest build is missing/broken.
func (upd *SyzUpdater) checkLatest() string {
if !osutil.FilesExist(upd.latestDir, syzFiles) {
return ""
}
tag, _ := readTag(filepath.Join(upd.latestDir, "tag"))
return tag
}
func readTag(file string) (tag string, mod time.Time) {
data, _ := ioutil.ReadFile(file)
tag = string(data)
if st, err := os.Stat(file); err == nil {
mod = st.ModTime()
}
if tag == "" || mod.IsZero() {
tag = ""
mod = time.Time{}
}
return
}
func formatTag(tag string) string {
if tag == "" {
return "unknown"
}
return tag
}

49
syz-ci/testdata/example.cfg vendored Normal file
View File

@ -0,0 +1,49 @@
{
"name": "ci",
"http": ":80",
"dashboard_addr": "1.2.3.4:1234",
"dashboard_key": "111",
"hub_addr": "2.3.4.5:2345",
"hub_key": "222",
"goroot": "/syzkaller/goroot",
"managers": [
{
"name": "upstream-kasan",
"repo": "git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git",
"branch": "master",
"compiler": "/syzkaller/gcc/bin/gcc",
"userspace": "/syzkaller/wheezy",
"kernel_config": "/syzkaller/kasan.config",
"manager_config": {
"http": ":10000",
"type": "gce",
"sandbox": "namespace",
"procs": 8,
"vm": {
"count": 10,
"machine_type": "n1-standard-2",
"gcs_bucket": "syzkaller"
}
}
},
{
"name": "linux-next-kasan",
"repo": "git://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git",
"branch": "master",
"compiler": "/syzkaller/gcc/bin/gcc",
"userspace": "/syzkaller/wheezy",
"kernel_config": "/syzkaller/kasan.config",
"manager_config": {
"http": ":10001",
"type": "gce",
"sandbox": "namespace",
"procs": 8,
"vm": {
"count": 10,
"machine_type": "n1-standard-2",
"gcs_bucket": "syzkaller"
}
}
}
]
}

14
syz-gce/syz-gce.go
View File

@ -354,16 +354,14 @@ func (a *LocalBuildAction) Build() error {
}
}
Logf(0, "building kernel on %v...", hash)
config, full := syzconfig, false
if cfg.Linux_Config != "" {
data, err := ioutil.ReadFile(cfg.Linux_Config)
if err != nil {
return fmt.Errorf("failed to read config file: %v", err)
if err := kernel.Build(dir, a.Compiler, cfg.Linux_Config); err != nil {
return fmt.Errorf("build failed: %v", err)
}
} else {
if err := kernel.BuildWithPartConfig(dir, a.Compiler, syzconfig); err != nil {
return fmt.Errorf("build failed: %v", err)
}
config, full = string(data), true
}
if err := kernel.Build(dir, a.Compiler, config, full); err != nil {
return fmt.Errorf("build failed: %v", err)
}
Logf(0, "building image...")
os.MkdirAll("image/obj", 0700)