vm/isolated: add initial support for fuzzing chromebooks

(WIP PR)

Add support for StartupScript.
* Modify Config{} to contain PostRepairScript.
* Allow repair() to execute a startup_script after reboot. The contents
of this script execute on the DUT.

Add pstore support:
* Modify Config{} to contain Pstore.
* Modify Diagnose() to reboot the DUT and fetch pstore logs,
conditional on inst.cfg.Pstore.
* Add readPstoreContents().
* Allow clearing previous pstore logs upon Create() and after use inside
readPstoreContents().
* Fetching pstore crashlogs relies on reliably getting lost connection
on DUT reboot. Use "ServerAliveInterval=6 ServerAliveCountMax=5" ssh
options when running syz-fuzzer with Pstore support enabled.

Allow parsing pstore contents:
* Diagnose() now returns pstore contents.

Refactoring:
* Move out some reusable parts of repair() to waitRebootAndSSH().
* Have an early return inside repair() if inst.waitForSSH() fails.
This commit is contained in:
Zubin Mithra 2020-02-06 13:58:50 -08:00 committed by Dmitry Vyukov
parent 81230308c6
commit 02698d8bc4
3 changed files with 223 additions and 36 deletions

View File

@ -53,6 +53,17 @@ Host *
Before fuzzing, connect to the machine and keep the connection open so all scp
and ssh usage will reuse it.
# Optional: Pstore support
If the device under test (DUT) has Pstore support, it is possible to configure syzkaller to
fetch crashlogs from /sys/fs/pstore. You can do this by setting `"pstore": true` within
the `vm` section of the syzkaller configuration file.
# Optional: Startup script
To execute commands on the DUT before fuzzing (re-)starts,
`startup_script` can be used.
## Syzkaller
Build syzkaller as described [here](/docs/contributing.md).
@ -71,6 +82,7 @@ Use the following config:
"type": "isolated",
"vm": {
"targets" : [ "10.0.0.1" ],
"pstore": false,
"target_dir" : "/home/user/tmp/syzkaller",
"target_reboot" : false
}

View File

@ -4,6 +4,7 @@
package isolated
import (
"bytes"
"fmt"
"io"
"io/ioutil"
@ -19,16 +20,20 @@ import (
"github.com/google/syzkaller/vm/vmimpl"
)
const pstoreConsoleFile = "/sys/fs/pstore/console-ramoops-0"
func init() {
vmimpl.Register("isolated", ctor, false)
}
type Config struct {
Host string `json:"host"` // host ip addr
Targets []string `json:"targets"` // target machines: (hostname|ip)(:port)?
TargetDir string `json:"target_dir"` // directory to copy/run on target
TargetReboot bool `json:"target_reboot"` // reboot target on repair
USBDevNums []string `json:"usb_device_num"` // /sys/bus/usb/devices/
Host string `json:"host"` // host ip addr
Targets []string `json:"targets"` // target machines: (hostname|ip)(:port)?
TargetDir string `json:"target_dir"` // directory to copy/run on target
TargetReboot bool `json:"target_reboot"` // reboot target on repair
USBDevNums []string `json:"usb_device_num"` // /sys/bus/usb/devices/
StartupScript string `json:"startup_script"` // script to execute after each startup
Pstore bool `json:"pstore"` // use crashlogs from pstore
}
type Pool struct {
@ -111,7 +116,7 @@ func (pool *Pool) Create(workdir string, index int) (vmimpl.Instance, error) {
}
}()
if err := inst.repair(); err != nil {
return nil, err
return nil, fmt.Errorf("repair failed: %v", err)
}
// Remount to writable.
@ -123,6 +128,11 @@ func (pool *Pool) Create(workdir string, index int) (vmimpl.Instance, error) {
// Remove temp files from previous runs.
inst.ssh("rm -rf '" + filepath.Join(inst.cfg.TargetDir, "*") + "'")
// Remove pstore files from previous runs.
if inst.cfg.Pstore {
inst.ssh(fmt.Sprintf("rm %v", pstoreConsoleFile))
}
closeInst = nil
return inst, nil
}
@ -189,40 +199,99 @@ func (inst *instance) ssh(command string) error {
return nil
}
func (inst *instance) waitRebootAndSSH(rebootTimeout int, sshTimeout time.Duration) error {
if err := inst.waitForReboot(rebootTimeout); err != nil {
log.Logf(2, "isolated: machine did not reboot")
return err
}
log.Logf(2, "isolated: rebooted wait for comeback")
if err := inst.waitForSSH(sshTimeout); err != nil {
log.Logf(2, "isolated: machine did not comeback")
return err
}
log.Logf(2, "isolated: reboot succeeded")
return nil
}
// Escapes double quotes(and nested double quote escapes). Ignores any other escapes.
// Reference: https://www.gnu.org/software/bash/manual/html_node/Double-Quotes.html
func escapeDoubleQuotes(inp string) string {
var ret strings.Builder
for pos := 0; pos < len(inp); pos++ {
// If inp[pos] is not a double quote or a backslash, just use
// as is.
if inp[pos] != '"' && inp[pos] != '\\' {
ret.WriteByte(inp[pos])
continue
}
// If it is a double quote, escape.
if inp[pos] == '"' {
ret.WriteString("\\\"")
continue
}
// If we detect a backslash, reescape only if what it's already escaping
// is a double-quotes.
temp := ""
j := pos
for ; j < len(inp); j++ {
if inp[j] == '\\' {
temp += string(inp[j])
continue
}
// If the escape corresponds to a double quotes, re-escape.
// Else, just use as is.
if inp[j] == '"' {
temp = temp + temp + "\\\""
} else {
temp += string(inp[j])
}
break
}
ret.WriteString(temp)
pos = j
}
return ret.String()
}
func (inst *instance) repair() error {
log.Logf(2, "isolated: trying to ssh")
if err := inst.waitForSSH(30 * time.Minute); err == nil {
if inst.cfg.TargetReboot {
if len(inst.cfg.USBDevNums) > 0 {
log.Logf(2, "isolated: trying to reboot by USB authorization")
usbAuth := fmt.Sprintf("%s%s%s", "/sys/bus/usb/devices/", inst.cfg.USBDevNums[inst.index], "/authorized")
if err := ioutil.WriteFile(usbAuth, []byte("0"), 0); err != nil {
log.Logf(2, "isolated: failed to turn off the device")
return err
}
if err := ioutil.WriteFile(usbAuth, []byte("1"), 0); err != nil {
log.Logf(2, "isolated: failed to turn on the device")
return err
}
} else {
log.Logf(2, "isolated: ssh succeeded, trying to reboot by ssh")
inst.ssh("reboot") // reboot will return an error, ignore it
}
}
if err := inst.waitForReboot(5 * 60); err != nil {
log.Logf(2, "isolated: machine did not reboot")
return err
}
log.Logf(2, "isolated: rebooted wait for comeback")
if err := inst.waitForSSH(30 * time.Minute); err != nil {
log.Logf(0, "isolated: machine did not comeback")
return err
}
log.Logf(2, "isolated: reboot succeeded")
} else {
if err := inst.waitForSSH(30 * time.Minute); err != nil {
log.Logf(2, "isolated: ssh failed")
return fmt.Errorf("SSH failed")
}
if inst.cfg.TargetReboot {
if len(inst.cfg.USBDevNums) > 0 {
log.Logf(2, "isolated: trying to reboot by USB authorization")
usbAuth := fmt.Sprintf("%s%s%s", "/sys/bus/usb/devices/", inst.cfg.USBDevNums[inst.index], "/authorized")
if err := ioutil.WriteFile(usbAuth, []byte("0"), 0); err != nil {
log.Logf(2, "isolated: failed to turn off the device")
return err
}
if err := ioutil.WriteFile(usbAuth, []byte("1"), 0); err != nil {
log.Logf(2, "isolated: failed to turn on the device")
return err
}
} else {
log.Logf(2, "isolated: ssh succeeded, trying to reboot by ssh")
inst.ssh("reboot") // reboot will return an error, ignore it
}
}
if err := inst.waitRebootAndSSH(5*60, 30*time.Minute); err != nil {
return fmt.Errorf("waitRebootAndSSH failed: %v", err)
}
if inst.cfg.StartupScript != "" {
log.Logf(2, "isolated: executing startup_script")
// Execute the contents of the StartupScript on the DUT.
contents, err := ioutil.ReadFile(inst.cfg.StartupScript)
if err != nil {
return fmt.Errorf("unable to read startup_script: %v", err)
}
c := string(contents)
if err := inst.ssh(fmt.Sprintf("bash -c \"%v\"", escapeDoubleQuotes(c))); err != nil {
return fmt.Errorf("failed to execute startup_script: %v", err)
}
log.Logf(2, "isolated: done executing startup_script")
}
return nil
}
@ -304,6 +373,10 @@ func (inst *instance) Run(timeout time.Duration, stop <-chan bool, command strin
proxy := fmt.Sprintf("%v:127.0.0.1:%v", inst.forwardPort, inst.forwardPort)
args = append(args, "-R", proxy)
}
if inst.cfg.Pstore {
args = append(args, "-o", "ServerAliveInterval=6")
args = append(args, "-o", "ServerAliveCountMax=5")
}
args = append(args, inst.sshUser+"@"+inst.targetAddr, "cd "+inst.cfg.TargetDir+" && exec "+command)
log.Logf(0, "running command: ssh %#v", args)
if inst.debug {
@ -331,8 +404,37 @@ func (inst *instance) Run(timeout time.Duration, stop <-chan bool, command strin
return vmimpl.Multiplex(cmd, merger, dmesg, timeout, stop, inst.closed, inst.debug)
}
func (inst *instance) readPstoreContents() ([]byte, error) {
log.Logf(0, "reading pstore contents")
args := append(vmimpl.SSHArgs(inst.debug, inst.sshKey, inst.targetPort),
inst.sshUser+"@"+inst.targetAddr, "cat "+pstoreConsoleFile+" && rm "+pstoreConsoleFile)
if inst.debug {
log.Logf(0, "running command: ssh %#v", args)
}
var stdout, stderr bytes.Buffer
cmd := osutil.Command("ssh", args...)
cmd.Stdout = &stdout
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
return nil, fmt.Errorf("unable to read pstore file: %v: %v", err, stderr.String())
}
return stdout.Bytes(), nil
}
func (inst *instance) Diagnose() ([]byte, bool) {
return nil, false
if !inst.cfg.Pstore {
return nil, false
}
log.Logf(2, "waiting for crashed DUT to come back up")
if err := inst.waitRebootAndSSH(5*60, 30*time.Minute); err != nil {
return []byte(fmt.Sprintf("unable to SSH into DUT after reboot: %v", err)), false
}
log.Logf(2, "reading contents of pstore")
contents, err := inst.readPstoreContents()
if err != nil {
return []byte(fmt.Sprintf("Diagnose failed: %v\n", err)), false
}
return contents, false
}
func splitTargetPort(addr string) (string, int, error) {

View File

@ -0,0 +1,73 @@
// Copyright 2020 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
package isolated
import "testing"
func TestEscapeDoubleQuotes(t *testing.T) {
testcases := []struct {
inp string
expected string
}{
// Input with no quoting returns the same string.
{
"",
"",
},
{
"adsf",
"adsf",
},
// Inputs with escaping of characters other that double
// quotes returns the same input.
{
"\\$\\`\\\\\n", // \$\`\\\n
"\\$\\`\\\\\n", // \$\`\\\n
},
// Input with double quote.
{
`"`,
`\"`,
},
// Input with already escaped double quote.
{
`\"`,
`\\\"`,
},
// Input with already escaped backtick and already
// double quote. Should only re-escape the
// double quote.
{
"\\`something\"", // \`something"
"\\`something\\\"", // \`something\"
},
// Input with already escaped backtick and already
// escaped double quote. Should only re-escape the
// escaped double quote.
{
"\\`something\\\"", // \`something\"
"\\`something\\\\\\\"", // \`something\\\"
},
{
`touch \
/tmp/OK
touch '/tmp/OK2'
touch "/tmp/OK3"
touch /tmp/OK4
bash -c "bash -c \"ls -al\""`,
`touch \
/tmp/OK
touch '/tmp/OK2'
touch \"/tmp/OK3\"
touch /tmp/OK4
bash -c \"bash -c \\\"ls -al\\\"\"`,
},
}
for i, tc := range testcases {
output := escapeDoubleQuotes(tc.inp)
if tc.expected != output {
t.Fatalf("%v: For input %v Expected escaped string %v got %v", i+1, tc.inp, tc.expected, output)
}
}
}