diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 93b92c5..3407565 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -19,3 +19,4 @@ Shuai Bai Alexander Popov Jean-Baptiste Cayrou Yuzhe Han +Thomas Garnier diff --git a/docs/setup.md b/docs/setup.md index cadc4ca..933db69 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -7,6 +7,7 @@ Instructions for a particular VM or kernel arch can be found on these pages: - [Setup: Ubuntu host, Odroid C2 board, arm64 kernel](setup_ubuntu-host_odroid-c2-board_arm64-kernel.md) - [Setup: Linux host, QEMU vm, arm64 kernel](setup_linux-host_qemu-vm_arm64-kernel.md) - [Setup: Linux host, Android device, arm64 kernel](setup_linux-host_android-device_arm64-kernel.md) +- [Setup: Linux isolated host](setup_linux-host_isolated.md) After following these instructions you should be able to run `syz-manager`, see it executing programs and be able to access statistics exposed at `http://127.0.0.1:56741`: diff --git a/docs/setup_linux-host_isolated.md b/docs/setup_linux-host_isolated.md new file mode 100644 index 0000000..7f26efd --- /dev/null +++ b/docs/setup_linux-host_isolated.md @@ -0,0 +1,108 @@ +# Setup: Linux isolated host + +These are the instructions on how to fuzz the kernel on isolated machines. +Isolated machines are separated in a way that limits remote management. They can +be interesting to fuzz due to specific hardware setups. + +This syzkaller configuration uses only ssh to launch and monitor an isolated +machine. + +## Setup reverse proxy support + +Given only ssh may work, a reverse ssh proxy will be used to allow the fuzzing +instance and the manager to communicate. + +Ensure the sshd configuration on the target machine has AllowTcpForwarding to yes. +``` +machine:~# grep Forwarding /etc/ssh/sshd_config +AllowTcpForwarding yes +``` + +## Kernel + +The isolated VM does not deploy kernel images so ensure the kernel on the target +machine is build with these options: +``` +CONFIG_KCOV=y +CONFIG_DEBUG_INFO=y +CONFIG_KASAN=y +CONFIG_KASAN_INLINE=y +``` + +Code coverage works better when KASLR Is disabled too: +``` +# CONFIG_RANDOMIZE_BASE is not set +``` + +## Optional: Reuse existing ssh connection + +In most scenarios, you should use an ssh key to connect to the target machine. +The isolated configuration supports ssh keys as described in the generic +[setup](setup_generic.md). + +If you cannot use an ssh key, you should configure your manager machine to reuse +existing ssh connections. + +Add these lines to your ~/.ssh/config file: +``` +Host * + ControlMaster auto + ControlPath ~/.ssh/control:%h:%p:%r +``` + +Before fuzzing, connect to the machine and keep the connection open so all scp +and ssh usage will reuse it. + +## Go + +Install Go 1.8.1: +``` bash +wget https://storage.googleapis.com/golang/go1.8.1.linux-amd64.tar.gz +tar -xf go1.8.1.linux-amd64.tar.gz +mv go goroot +export GOROOT=`pwd`/goroot +export PATH=$PATH:$GOROOT/bin +mkdir gopath +export GOPATH=`pwd`/gopath +``` + +## Syzkaller + +Get and build syzkaller: +``` bash +go get -u -d github.com/google/syzkaller/... +cd gopath/src/github.com/google/syzkaller/ +make +``` + +Use the following config: +``` +{ + "http": "127.0.0.1:56741", + "rpc": "127.0.0.1:0", + "sshkey" : "/path/to/optional/sshkey", + "workdir": "/syzkaller/workdir", + "vmlinux": "/linux-next/vmlinux", + "syzkaller": "/go/src/github.com/google/syzkaller", + "sandbox": "setuid", + "type": "isolated", + "vm": { + "targets" : [ "10.0.0.1" ], + "target_dir" : "/home/user/tmp/syzkaller", + "target_reboot" : false, + } +} +``` + +Don't forget to update: + - `workdir` (path to the workdir) + - `vmlinux` (path to the `vmlinux` binary) + - `sshkey` You can setup an sshkey (optional) + - `vm.targets` List of hosts to use for fufzzing + - `vm.target_dir` Working directory on the target host + - `vm.target_reboot` Reboot the machine if remote process hang (useful for wide fuzzing, false by default) + +Run syzkaller manager: +``` bash +./bin/syz-manager -config=my.cfg +``` diff --git a/vm/isolated/isolated.go b/vm/isolated/isolated.go new file mode 100644 index 0000000..20c09c7 --- /dev/null +++ b/vm/isolated/isolated.go @@ -0,0 +1,359 @@ +// Copyright 2017 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package isolated + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "time" + + "github.com/google/syzkaller/pkg/config" + . "github.com/google/syzkaller/pkg/log" + "github.com/google/syzkaller/pkg/osutil" + "github.com/google/syzkaller/vm/vmimpl" +) + +func init() { + vmimpl.Register("isolated", ctor) +} + +type Config struct { + Targets []string // target machines + Target_Dir string // directory to copy/run on target + Target_Reboot bool // reboot target on repair +} + +type Pool struct { + env *vmimpl.Env + cfg *Config +} + +type instance struct { + cfg *Config + target string + closed chan bool + debug bool + sshkey string + port int +} + +func ctor(env *vmimpl.Env) (vmimpl.Pool, error) { + cfg := &Config{} + if err := config.LoadData(env.Config, cfg); err != nil { + return nil, err + } + if len(cfg.Targets) == 0 { + return nil, fmt.Errorf("config param targets is empty") + } + if cfg.Target_Dir == "" { + return nil, fmt.Errorf("config param target_dir is empty") + } + // sshkey is optional + if env.Sshkey != "" && !osutil.IsExist(env.Sshkey) { + return nil, fmt.Errorf("ssh key '%v' does not exist", env.Sshkey) + } + if env.Debug { + cfg.Targets = cfg.Targets[:1] + } + pool := &Pool{ + cfg: cfg, + env: env, + } + return pool, nil +} + +func (pool *Pool) Count() int { + return len(pool.cfg.Targets) +} + +func (pool *Pool) Create(workdir string, index int) (vmimpl.Instance, error) { + inst := &instance{ + cfg: pool.cfg, + target: pool.cfg.Targets[index], + closed: make(chan bool), + debug: pool.env.Debug, + sshkey: pool.env.Sshkey, + } + closeInst := inst + defer func() { + if closeInst != nil { + closeInst.Close() + } + }() + if err := inst.repair(); err != nil { + return nil, err + } + + // Create working dir if doesn't exist. + inst.ssh("mkdir -p '" + inst.cfg.Target_Dir + "'") + + // Remove temp files from previous runs. + inst.ssh("rm -rf '" + filepath.Join(inst.cfg.Target_Dir, "*") + "'") + + closeInst = nil + return inst, nil +} + +func (inst *instance) Forward(port int) (string, error) { + if inst.port != 0 { + return "", fmt.Errorf("isolated: Forward port already set") + } + if port == 0 { + return "", fmt.Errorf("isolated: Forward port is zero") + } + inst.port = port + return fmt.Sprintf("127.0.0.1:%v", port), nil +} + +func (inst *instance) ssh(command string) ([]byte, error) { + if inst.debug { + Logf(0, "executing ssh %+v", command) + } + + rpipe, wpipe, err := osutil.LongPipe() + if err != nil { + return nil, err + } + + args := append(inst.sshArgs("-p"), "root@"+inst.target, command) + if inst.debug { + Logf(0, "running command: ssh %#v", args) + } + cmd := exec.Command("ssh", args...) + cmd.Stdout = wpipe + cmd.Stderr = wpipe + if err := cmd.Start(); err != nil { + wpipe.Close() + return nil, err + } + wpipe.Close() + + done := make(chan bool) + go func() { + select { + case <-time.After(time.Minute): + if inst.debug { + Logf(0, "ssh hanged") + } + cmd.Process.Kill() + case <-done: + } + }() + if err := cmd.Wait(); err != nil { + close(done) + out, _ := ioutil.ReadAll(rpipe) + if inst.debug { + Logf(0, "ssh failed: %v\n%s", err, out) + } + return nil, fmt.Errorf("ssh %+v failed: %v\n%s", args, err, out) + } + close(done) + if inst.debug { + Logf(0, "ssh returned") + } + out, _ := ioutil.ReadAll(rpipe) + return out, nil +} + +func (inst *instance) repair() error { + Logf(2, "isolated: trying to ssh") + if err := inst.waitForSsh(30 * 60); err == nil { + Logf(2, "isolated: ssh succeeded") + if inst.cfg.Target_Reboot == true { + if _, err = inst.ssh("reboot"); err != nil { + Logf(2, "isolated: failed to send reboot command") + return err + } + if err := inst.waitForReboot(5 * 60); err != nil { + Logf(2, "isolated: machine did not reboot") + return err + } + if err := inst.waitForSsh(30 * 60); err != nil { + Logf(2, "isolated: machine did not comeback") + return err + } + } + } else { + Logf(2, "isolated: ssh failed") + return fmt.Errorf("SSH failed") + } + + return nil +} + +func (inst *instance) waitForSsh(timeout int) error { + var err error + start := time.Now() + for { + if !vmimpl.SleepInterruptible(time.Second) { + return fmt.Errorf("shutdown in progress") + } + if _, err = inst.ssh("pwd"); err == nil { + return nil + } + if time.Since(start).Seconds() > float64(timeout) { + break + } + } + return fmt.Errorf("isolated: instance is dead and unrepairable: %v", err) +} + +func (inst *instance) waitForReboot(timeout int) error { + var err error + start := time.Now() + for { + if !vmimpl.SleepInterruptible(time.Second) { + return fmt.Errorf("shutdown in progress") + } + // If it fails, then the reboot started + if _, err = inst.ssh("pwd"); err != nil { + return nil + } + if time.Since(start).Seconds() > float64(timeout) { + break + } + } + return fmt.Errorf("isolated: the machine did not reboot on repair") +} + + +func (inst *instance) Close() { + close(inst.closed) +} + +func (inst *instance) Copy(hostSrc string) (string, error) { + baseName := filepath.Base(hostSrc) + vmDst := filepath.Join(inst.cfg.Target_Dir, baseName) + inst.ssh("pkill -9 '" + baseName + "'; rm -f '" + vmDst + "'") + args := append(inst.sshArgs("-P"), hostSrc, "root@"+inst.target+":"+vmDst) + cmd := exec.Command("scp", args...) + if inst.debug { + Logf(0, "running command: scp %#v", args) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stdout + } + if err := cmd.Start(); err != nil { + return "", err + } + done := make(chan bool) + go func() { + select { + case <-time.After(3 * time.Minute): + cmd.Process.Kill() + case <-done: + } + }() + err := cmd.Wait() + close(done) + if err != nil { + return "", err + } + return vmDst, nil +} + +func (inst *instance) Run(timeout time.Duration, stop <-chan bool, command string) (<-chan []byte, <-chan error, error) { + args := append(inst.sshArgs("-p"), "root@"+inst.target) + dmesg, err := vmimpl.OpenRemoteConsole("ssh", args...) + if err != nil { + return nil, nil, err + } + + rpipe, wpipe, err := osutil.LongPipe() + if err != nil { + dmesg.Close() + return nil, nil, err + } + + args = inst.sshArgs("-p") + // Forward target port as part of the ssh connection (reverse proxy) + if inst.port != 0 { + proxy := fmt.Sprintf("%v:127.0.0.1:%v", inst.port, inst.port) + args = append(args, "-R", proxy) + } + args = append(args, "root@"+inst.target, "cd "+inst.cfg.Target_Dir+" && exec "+command) + Logf(0, "running command: ssh %#v", args) + if inst.debug { + Logf(0, "running command: ssh %#v", args) + } + cmd := exec.Command("ssh", args...) + cmd.Stdout = wpipe + cmd.Stderr = wpipe + if err := cmd.Start(); err != nil { + dmesg.Close() + rpipe.Close() + wpipe.Close() + return nil, nil, err + } + wpipe.Close() + + var tee io.Writer + if inst.debug { + tee = os.Stdout + } + merger := vmimpl.NewOutputMerger(tee) + merger.Add("dmesg", dmesg) + merger.Add("ssh", rpipe) + + errc := make(chan error, 1) + signal := func(err error) { + select { + case errc <- err: + default: + } + } + + go func() { + select { + case <-time.After(timeout): + signal(vmimpl.TimeoutErr) + case <-stop: + signal(vmimpl.TimeoutErr) + case <-inst.closed: + if inst.debug { + Logf(0, "instance closed") + } + signal(fmt.Errorf("instance closed")) + case err := <-merger.Err: + cmd.Process.Kill() + dmesg.Close() + merger.Wait() + if cmdErr := cmd.Wait(); cmdErr == nil { + // If the command exited successfully, we got EOF error from merger. + // But in this case no error has happened and the EOF is expected. + err = nil + } + signal(err) + return + } + cmd.Process.Kill() + dmesg.Close() + merger.Wait() + cmd.Wait() + }() + return merger.Output, errc, nil +} + +func (inst *instance) sshArgs(portArg string) []string { + args := []string{ + portArg, "22", + "-o", "ConnectionAttempts=10", + "-o", "ConnectTimeout=10", + "-o", "BatchMode=yes", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "IdentitiesOnly=yes", + "-o", "StrictHostKeyChecking=no", + "-o", "LogLevel=error", + } + if inst.sshkey != "" { + args = append(args, "-i", inst.sshkey) + } + if inst.debug { + args = append(args, "-v") + } + return args +} diff --git a/vm/vm.go b/vm/vm.go index 74eca21..82cb342 100644 --- a/vm/vm.go +++ b/vm/vm.go @@ -21,6 +21,7 @@ import ( _ "github.com/google/syzkaller/vm/adb" _ "github.com/google/syzkaller/vm/gce" + _ "github.com/google/syzkaller/vm/isolated" _ "github.com/google/syzkaller/vm/kvm" _ "github.com/google/syzkaller/vm/odroid" _ "github.com/google/syzkaller/vm/qemu" diff --git a/vm/vmimpl/console.go b/vm/vmimpl/console.go index 7fe878c..7059221 100644 --- a/vm/vmimpl/console.go +++ b/vm/vmimpl/console.go @@ -76,13 +76,14 @@ func (t *tty) Close() error { return nil } -// OpenAdbConsole provides fallback console output using 'adb shell dmesg -w'. -func OpenAdbConsole(bin, dev string) (rc io.ReadCloser, err error) { +// Open dmesg remotely +func OpenRemoteConsole(bin string, args ...string) (rc io.ReadCloser, err error) { rpipe, wpipe, err := osutil.LongPipe() if err != nil { return nil, err } - cmd := exec.Command(bin, "-s", dev, "shell", "dmesg -w") + args = append(args, "dmesg -w") + cmd := exec.Command(bin, args...) cmd.Stdout = wpipe cmd.Stderr = wpipe if err := cmd.Start(); err != nil { @@ -91,28 +92,33 @@ func OpenAdbConsole(bin, dev string) (rc io.ReadCloser, err error) { return nil, fmt.Errorf("failed to start adb: %v", err) } wpipe.Close() - con := &adbCon{ + con := &remoteCon{ cmd: cmd, rpipe: rpipe, } return con, err } -type adbCon struct { +// OpenAdbConsole provides fallback console output using 'adb shell dmesg -w'. +func OpenAdbConsole(bin, dev string) (rc io.ReadCloser, err error) { + return OpenRemoteConsole(bin, "-s", dev, "shell") +} + +type remoteCon struct { closeMu sync.Mutex readMu sync.Mutex cmd *exec.Cmd rpipe io.ReadCloser } -func (t *adbCon) Read(buf []byte) (int, error) { +func (t *remoteCon) Read(buf []byte) (int, error) { t.readMu.Lock() n, err := t.rpipe.Read(buf) t.readMu.Unlock() return n, err } -func (t *adbCon) Close() error { +func (t *remoteCon) Close() error { t.closeMu.Lock() cmd := t.cmd t.cmd = nil