Add Isolated VM

Add a new isolated VM for machines that you cannot easily manage. It
assumes the machine is only available through SSH and create a reverse
proxy to ensure the machine can connect back to syz-manager.

Signed-off-by: Thomas Garnier <thgarnie@google.com>

CONTRIBUTORS

@@ -19,3 +19,4 @@ Shuai Bai
 Alexander Popov
 Jean-Baptiste Cayrou
 Yuzhe Han
+Thomas Garnier

docs/setup.md

@@ -7,6 +7,7 @@ Instructions for a particular VM or kernel arch can be found on these pages:
 - [Setup: Ubuntu host, Odroid C2 board, arm64 kernel](setup_ubuntu-host_odroid-c2-board_arm64-kernel.md)
 - [Setup: Linux host, QEMU vm, arm64 kernel](setup_linux-host_qemu-vm_arm64-kernel.md)
 - [Setup: Linux host, Android device, arm64 kernel](setup_linux-host_android-device_arm64-kernel.md)
+- [Setup: Linux isolated host](setup_linux-host_isolated.md)
 
 After following these instructions you should be able to run `syz-manager`, see it executing programs and be able to access statistics exposed at `http://127.0.0.1:56741`:
 

docs/setup_linux-host_isolated.md

@@ -0,0 +1,108 @@
+# Setup: Linux isolated host
+
+These are the instructions on how to fuzz the kernel on isolated machines.
+Isolated machines are separated in a way that limits remote management. They can
+be interesting to fuzz due to specific hardware setups.
+
+This syzkaller configuration uses only ssh to launch and monitor an isolated
+machine.
+
+## Setup reverse proxy support
+
+Given only ssh may work, a reverse ssh proxy will be used to allow the fuzzing
+instance and the manager to communicate.
+
+Ensure the sshd configuration on the target machine has AllowTcpForwarding to yes.
+```
+machine:~# grep Forwarding /etc/ssh/sshd_config
+AllowTcpForwarding yes
+```
+
+## Kernel
+
+The isolated VM does not deploy kernel images so ensure the kernel on the target
+machine is build with these options:
+```
+CONFIG_KCOV=y
+CONFIG_DEBUG_INFO=y
+CONFIG_KASAN=y
+CONFIG_KASAN_INLINE=y
+```
+
+Code coverage works better when KASLR Is disabled too:
+```
+# CONFIG_RANDOMIZE_BASE is not set
+```
+
+## Optional: Reuse existing ssh connection
+
+In most scenarios, you should use an ssh key to connect to the target machine.
+The isolated configuration supports ssh keys as described in the generic
+[setup](setup_generic.md).
+
+If you cannot use an ssh key, you should configure your manager machine to reuse
+existing ssh connections.
+
+Add these lines to your ~/.ssh/config file:
+```
+Host *
+	ControlMaster auto
+	ControlPath ~/.ssh/control:%h:%p:%r
+```
+
+Before fuzzing, connect to the machine and keep the connection open so all scp
+and ssh usage will reuse it.
+
+## Go
+
+Install Go 1.8.1:
+``` bash
+wget https://storage.googleapis.com/golang/go1.8.1.linux-amd64.tar.gz
+tar -xf go1.8.1.linux-amd64.tar.gz
+mv go goroot
+export GOROOT=`pwd`/goroot
+export PATH=$PATH:$GOROOT/bin
+mkdir gopath
+export GOPATH=`pwd`/gopath
+```
+
+## Syzkaller
+
+Get and build syzkaller:
+``` bash
+go get -u -d github.com/google/syzkaller/...
+cd gopath/src/github.com/google/syzkaller/
+make
+```
+
+Use the following config:
+```
+{
+	"http": "127.0.0.1:56741",
+	"rpc": "127.0.0.1:0",
+	"sshkey" : "/path/to/optional/sshkey",
+	"workdir": "/syzkaller/workdir",
+	"vmlinux": "/linux-next/vmlinux",
+	"syzkaller": "/go/src/github.com/google/syzkaller",
+	"sandbox": "setuid",
+	"type": "isolated",
+	"vm": {
+		"targets" : [ "10.0.0.1" ],
+		"target_dir" : "/home/user/tmp/syzkaller",
+                "target_reboot" : false,
+	}
+}
+```
+
+Don't forget to update:
+ - `workdir` (path to the workdir)
+ - `vmlinux` (path to the `vmlinux` binary)
+ - `sshkey` You can setup an sshkey (optional)
+ - `vm.targets` List of hosts to use for fufzzing
+ - `vm.target_dir` Working directory on the target host
+ - `vm.target_reboot` Reboot the machine if remote process hang (useful for wide fuzzing, false by default)
+
+Run syzkaller manager:
+``` bash
+./bin/syz-manager -config=my.cfg
+```

vm/isolated/isolated.go

@@ -0,0 +1,359 @@
+// Copyright 2017 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package isolated
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"time"
+
+	"github.com/google/syzkaller/pkg/config"
+	. "github.com/google/syzkaller/pkg/log"
+	"github.com/google/syzkaller/pkg/osutil"
+	"github.com/google/syzkaller/vm/vmimpl"
+)
+
+func init() {
+	vmimpl.Register("isolated", ctor)
+}
+
+type Config struct {
+	Targets       []string // target machines
+	Target_Dir    string   // directory to copy/run on target
+	Target_Reboot bool     // reboot target on repair
+}
+
+type Pool struct {
+	env *vmimpl.Env
+	cfg *Config
+}
+
+type instance struct {
+	cfg    *Config
+	target string
+	closed chan bool
+	debug  bool
+	sshkey string
+	port   int
+}
+
+func ctor(env *vmimpl.Env) (vmimpl.Pool, error) {
+	cfg := &Config{}
+	if err := config.LoadData(env.Config, cfg); err != nil {
+		return nil, err
+	}
+	if len(cfg.Targets) == 0 {
+		return nil, fmt.Errorf("config param targets is empty")
+	}
+	if cfg.Target_Dir == "" {
+		return nil, fmt.Errorf("config param target_dir is empty")
+	}
+	// sshkey is optional
+	if env.Sshkey != "" && !osutil.IsExist(env.Sshkey) {
+		return nil, fmt.Errorf("ssh key '%v' does not exist", env.Sshkey)
+	}
+	if env.Debug {
+		cfg.Targets = cfg.Targets[:1]
+	}
+	pool := &Pool{
+		cfg: cfg,
+		env: env,
+	}
+	return pool, nil
+}
+
+func (pool *Pool) Count() int {
+	return len(pool.cfg.Targets)
+}
+
+func (pool *Pool) Create(workdir string, index int) (vmimpl.Instance, error) {
+	inst := &instance{
+		cfg:    pool.cfg,
+		target: pool.cfg.Targets[index],
+		closed: make(chan bool),
+		debug:  pool.env.Debug,
+		sshkey: pool.env.Sshkey,
+	}
+	closeInst := inst
+	defer func() {
+		if closeInst != nil {
+			closeInst.Close()
+		}
+	}()
+	if err := inst.repair(); err != nil {
+		return nil, err
+	}
+
+	// Create working dir if doesn't exist.
+	inst.ssh("mkdir -p '" + inst.cfg.Target_Dir + "'")
+
+	// Remove temp files from previous runs.
+	inst.ssh("rm -rf '" + filepath.Join(inst.cfg.Target_Dir, "*") + "'")
+
+	closeInst = nil
+	return inst, nil
+}
+
+func (inst *instance) Forward(port int) (string, error) {
+	if inst.port != 0 {
+		return "", fmt.Errorf("isolated: Forward port already set")
+	}
+	if port == 0 {
+		return "", fmt.Errorf("isolated: Forward port is zero")
+	}
+	inst.port = port
+	return fmt.Sprintf("127.0.0.1:%v", port), nil
+}
+
+func (inst *instance) ssh(command string) ([]byte, error) {
+	if inst.debug {
+		Logf(0, "executing ssh %+v", command)
+	}
+
+	rpipe, wpipe, err := osutil.LongPipe()
+	if err != nil {
+		return nil, err
+	}
+
+	args := append(inst.sshArgs("-p"), "root@"+inst.target, command)
+	if inst.debug {
+		Logf(0, "running command: ssh %#v", args)
+	}
+	cmd := exec.Command("ssh", args...)
+	cmd.Stdout = wpipe
+	cmd.Stderr = wpipe
+	if err := cmd.Start(); err != nil {
+		wpipe.Close()
+		return nil, err
+	}
+	wpipe.Close()
+
+	done := make(chan bool)
+	go func() {
+		select {
+		case <-time.After(time.Minute):
+			if inst.debug {
+				Logf(0, "ssh hanged")
+			}
+			cmd.Process.Kill()
+		case <-done:
+		}
+	}()
+	if err := cmd.Wait(); err != nil {
+		close(done)
+		out, _ := ioutil.ReadAll(rpipe)
+		if inst.debug {
+			Logf(0, "ssh failed: %v\n%s", err, out)
+		}
+		return nil, fmt.Errorf("ssh %+v failed: %v\n%s", args, err, out)
+	}
+	close(done)
+	if inst.debug {
+		Logf(0, "ssh returned")
+	}
+	out, _ := ioutil.ReadAll(rpipe)
+	return out, nil
+}
+
+func (inst *instance) repair() error {
+	Logf(2, "isolated: trying to ssh")
+	if err := inst.waitForSsh(30 * 60); err == nil {
+		Logf(2, "isolated: ssh succeeded")
+		if inst.cfg.Target_Reboot == true {
+			if _, err = inst.ssh("reboot"); err != nil {
+				Logf(2, "isolated: failed to send reboot command")
+				return err
+			}
+			if err := inst.waitForReboot(5 * 60); err != nil {
+				Logf(2, "isolated: machine did not reboot")
+				return err
+			}
+			if err := inst.waitForSsh(30 * 60); err != nil {
+				Logf(2, "isolated: machine did not comeback")
+				return err
+			}
+		}
+	} else {
+		Logf(2, "isolated: ssh failed")
+		return fmt.Errorf("SSH failed")
+	}
+
+	return nil
+}
+
+func (inst *instance) waitForSsh(timeout int) error {
+	var err error
+	start := time.Now()
+	for {
+		if !vmimpl.SleepInterruptible(time.Second) {
+			return fmt.Errorf("shutdown in progress")
+		}
+		if _, err = inst.ssh("pwd"); err == nil {
+			return nil
+		}
+		if time.Since(start).Seconds() > float64(timeout) {
+			break
+		}
+	}
+	return fmt.Errorf("isolated: instance is dead and unrepairable: %v", err)
+}
+
+func (inst *instance) waitForReboot(timeout int) error {
+	var err error
+	start := time.Now()
+	for {
+		if !vmimpl.SleepInterruptible(time.Second) {
+			return fmt.Errorf("shutdown in progress")
+		}
+		// If it fails, then the reboot started
+		if _, err = inst.ssh("pwd"); err != nil {
+			return nil
+		}
+		if time.Since(start).Seconds() > float64(timeout) {
+			break
+		}
+	}
+	return fmt.Errorf("isolated: the machine did not reboot on repair")
+}
+
+
+func (inst *instance) Close() {
+	close(inst.closed)
+}
+
+func (inst *instance) Copy(hostSrc string) (string, error) {
+	baseName := filepath.Base(hostSrc)
+	vmDst := filepath.Join(inst.cfg.Target_Dir, baseName)
+	inst.ssh("pkill -9 '" + baseName + "'; rm -f '" + vmDst + "'")
+	args := append(inst.sshArgs("-P"), hostSrc, "root@"+inst.target+":"+vmDst)
+	cmd := exec.Command("scp", args...)
+	if inst.debug {
+		Logf(0, "running command: scp %#v", args)
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stdout
+	}
+	if err := cmd.Start(); err != nil {
+		return "", err
+	}
+	done := make(chan bool)
+	go func() {
+		select {
+		case <-time.After(3 * time.Minute):
+			cmd.Process.Kill()
+		case <-done:
+		}
+	}()
+	err := cmd.Wait()
+	close(done)
+	if err != nil {
+		return "", err
+	}
+	return vmDst, nil
+}
+
+func (inst *instance) Run(timeout time.Duration, stop <-chan bool, command string) (<-chan []byte, <-chan error, error) {
+	args := append(inst.sshArgs("-p"), "root@"+inst.target)
+	dmesg, err := vmimpl.OpenRemoteConsole("ssh", args...)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	rpipe, wpipe, err := osutil.LongPipe()
+	if err != nil {
+		dmesg.Close()
+		return nil, nil, err
+	}
+
+	args = inst.sshArgs("-p")
+	// Forward target port as part of the ssh connection (reverse proxy)
+	if inst.port != 0 {
+		proxy := fmt.Sprintf("%v:127.0.0.1:%v", inst.port, inst.port)
+		args = append(args, "-R", proxy)
+	}
+	args = append(args, "root@"+inst.target, "cd "+inst.cfg.Target_Dir+" && exec "+command)
+	Logf(0, "running command: ssh %#v", args)
+	if inst.debug {
+		Logf(0, "running command: ssh %#v", args)
+	}
+	cmd := exec.Command("ssh", args...)
+	cmd.Stdout = wpipe
+	cmd.Stderr = wpipe
+	if err := cmd.Start(); err != nil {
+		dmesg.Close()
+		rpipe.Close()
+		wpipe.Close()
+		return nil, nil, err
+	}
+	wpipe.Close()
+
+	var tee io.Writer
+	if inst.debug {
+		tee = os.Stdout
+	}
+	merger := vmimpl.NewOutputMerger(tee)
+	merger.Add("dmesg", dmesg)
+	merger.Add("ssh", rpipe)
+
+	errc := make(chan error, 1)
+	signal := func(err error) {
+		select {
+		case errc <- err:
+		default:
+		}
+	}
+
+	go func() {
+		select {
+		case <-time.After(timeout):
+			signal(vmimpl.TimeoutErr)
+		case <-stop:
+			signal(vmimpl.TimeoutErr)
+		case <-inst.closed:
+			if inst.debug {
+				Logf(0, "instance closed")
+			}
+			signal(fmt.Errorf("instance closed"))
+		case err := <-merger.Err:
+			cmd.Process.Kill()
+			dmesg.Close()
+			merger.Wait()
+			if cmdErr := cmd.Wait(); cmdErr == nil {
+				// If the command exited successfully, we got EOF error from merger.
+				// But in this case no error has happened and the EOF is expected.
+				err = nil
+			}
+			signal(err)
+			return
+		}
+		cmd.Process.Kill()
+		dmesg.Close()
+		merger.Wait()
+		cmd.Wait()
+	}()
+	return merger.Output, errc, nil
+}
+
+func (inst *instance) sshArgs(portArg string) []string {
+	args := []string{
+		portArg, "22",
+		"-o", "ConnectionAttempts=10",
+		"-o", "ConnectTimeout=10",
+		"-o", "BatchMode=yes",
+		"-o", "UserKnownHostsFile=/dev/null",
+		"-o", "IdentitiesOnly=yes",
+		"-o", "StrictHostKeyChecking=no",
+		"-o", "LogLevel=error",
+	}
+	if inst.sshkey != "" {
+		args = append(args, "-i", inst.sshkey)
+	}
+	if inst.debug {
+		args = append(args, "-v")
+	}
+	return args
+}

vm/vm.go

@@ -21,6 +21,7 @@ import (
 
 	_ "github.com/google/syzkaller/vm/adb"
 	_ "github.com/google/syzkaller/vm/gce"
+	_ "github.com/google/syzkaller/vm/isolated"
 	_ "github.com/google/syzkaller/vm/kvm"
 	_ "github.com/google/syzkaller/vm/odroid"
 	_ "github.com/google/syzkaller/vm/qemu"

vm/vmimpl/console.go

@@ -76,13 +76,14 @@ func (t *tty) Close() error {
 	return nil
 }
 
-// OpenAdbConsole provides fallback console output using 'adb shell dmesg -w'.
+// Open dmesg remotely
-func OpenAdbConsole(bin, dev string) (rc io.ReadCloser, err error) {
+func OpenRemoteConsole(bin string, args ...string) (rc io.ReadCloser, err error) {
 	rpipe, wpipe, err := osutil.LongPipe()
 	if err != nil {
 		return nil, err
 	}
-	cmd := exec.Command(bin, "-s", dev, "shell", "dmesg -w")
+	args = append(args, "dmesg -w")
+	cmd := exec.Command(bin, args...)
 	cmd.Stdout = wpipe
 	cmd.Stderr = wpipe
 	if err := cmd.Start(); err != nil {
@@ -91,28 +92,33 @@ func OpenAdbConsole(bin, dev string) (rc io.ReadCloser, err error) {
 		return nil, fmt.Errorf("failed to start adb: %v", err)
 	}
 	wpipe.Close()
-	con := &adbCon{
+	con := &remoteCon{
 		cmd:   cmd,
 		rpipe: rpipe,
 	}
 	return con, err
 }
 
-type adbCon struct {
+// OpenAdbConsole provides fallback console output using 'adb shell dmesg -w'.
+func OpenAdbConsole(bin, dev string) (rc io.ReadCloser, err error) {
+	return OpenRemoteConsole(bin, "-s", dev, "shell")
+}
+
+type remoteCon struct {
 	closeMu sync.Mutex
 	readMu  sync.Mutex
 	cmd     *exec.Cmd
 	rpipe   io.ReadCloser
 }
 
-func (t *adbCon) Read(buf []byte) (int, error) {
+func (t *remoteCon) Read(buf []byte) (int, error) {
 	t.readMu.Lock()
 	n, err := t.rpipe.Read(buf)
 	t.readMu.Unlock()
 	return n, err
 }
 
-func (t *adbCon) Close() error {
+func (t *remoteCon) Close() error {
 	t.closeMu.Lock()
 	cmd := t.cmd
 	t.cmd = nil