manager: precreate crashes dir

http handler scrapes crashes dir, it becomes upset if the dir is missing

gce/gce.go

@@ -0,0 +1,218 @@
+// Copyright 2016 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+// Package gce provides wrappers around Google Compute Engine (GCE) APIs.
+// It is assumed that the program itself also runs on GCE as APIs operate on the current project/zone.
+package gce
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"time"
+
+	"golang.org/x/net/context"
+	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/google"
+	"google.golang.org/api/compute/v0.beta"
+	"google.golang.org/api/googleapi"
+)
+
+type Context struct {
+	ProjectID  string
+	ZoneID     string
+	Instance   string
+	InternalIP string
+
+	computeService *compute.Service
+
+	// apiCallTicker ticks regularly, preventing us from accidentally making
+	// GCE API calls too quickly. Our quota is 20 QPS, but we temporarily
+	// limit ourselves to less than that.
+	apiRateGate <-chan time.Time
+}
+
+func NewContext() (*Context, error) {
+	ctx := &Context{
+		apiRateGate: time.NewTicker(time.Second / 10).C,
+	}
+	background := context.Background()
+	tokenSource, err := google.DefaultTokenSource(background, compute.CloudPlatformScope)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get a token source: %v", err)
+	}
+	httpClient := oauth2.NewClient(background, tokenSource)
+	ctx.computeService, _ = compute.New(httpClient)
+	// Obtain project name, zone and current instance IP address.
+	ctx.ProjectID, err = ctx.getMeta("project/project-id")
+	if err != nil {
+		return nil, fmt.Errorf("failed to query gce project-id: %v", err)
+	}
+	ctx.ZoneID, err = ctx.getMeta("instance/zone")
+	if err != nil {
+		return nil, fmt.Errorf("failed to query gce zone: %v", err)
+	}
+	if i := strings.LastIndexByte(ctx.ZoneID, '/'); i != -1 {
+		ctx.ZoneID = ctx.ZoneID[i+1:] // the query returns some nonsense prefix
+	}
+	instID, err := ctx.getMeta("instance/id")
+	if err != nil {
+		return nil, fmt.Errorf("failed to query gce instance id: %v", err)
+	}
+	instances, err := ctx.computeService.Instances.List(ctx.ProjectID, ctx.ZoneID).Do()
+	if err != nil {
+		return nil, fmt.Errorf("error getting instance list: %v", err)
+	}
+	// Finds this instance internal IP.
+	for _, inst := range instances.Items {
+		if fmt.Sprint(inst.Id) != instID {
+			continue
+		}
+		ctx.Instance = inst.Name
+		for _, iface := range inst.NetworkInterfaces {
+			if strings.HasPrefix(iface.NetworkIP, "10.") {
+				ctx.InternalIP = iface.NetworkIP
+				break
+			}
+		}
+		break
+	}
+	if ctx.Instance == "" || ctx.InternalIP == "" {
+		return nil, fmt.Errorf("failed to get current instance name and internal IP")
+	}
+	return ctx, nil
+}
+
+func (ctx *Context) getMeta(path string) (string, error) {
+	req, err := http.NewRequest("GET", "http://metadata.google.internal/computeMetadata/v1/"+path, nil)
+	if err != nil {
+		return "", err
+	}
+	req.Header.Add("Metadata-Flavor", "Google")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+	return string(body), nil
+}
+
+func (ctx *Context) CreateInstance(name, machineType, image, sshkey string) (string, error) {
+	prefix := "https://www.googleapis.com/compute/v1/projects/" + ctx.ProjectID
+	instance := &compute.Instance{
+		Name:        name,
+		Description: "syzkaller worker",
+		MachineType: prefix + "/zones/" + ctx.ZoneID + "/machineTypes/" + machineType,
+		Disks: []*compute.AttachedDisk{
+			{
+				AutoDelete: true,
+				Boot:       true,
+				Type:       "PERSISTENT",
+				InitializeParams: &compute.AttachedDiskInitializeParams{
+					DiskName:    name,
+					SourceImage: prefix + "/global/images/" + image,
+				},
+			},
+		},
+		Metadata: &compute.Metadata{
+			Items: []*compute.MetadataItems{
+				{
+					Key:   "ssh-keys",
+					Value: "syzkaller:" + sshkey,
+				},
+				{
+					Key:   "serial-port-enable",
+					Value: "1",
+				},
+			},
+		},
+		NetworkInterfaces: []*compute.NetworkInterface{
+			&compute.NetworkInterface{
+				Network: "global/networks/default",
+			},
+		},
+		Scheduling: &compute.Scheduling{
+			AutomaticRestart:  false,
+			Preemptible:       false,
+			OnHostMaintenance: "MIGRATE",
+		},
+	}
+
+	<-ctx.apiRateGate
+	op, err := ctx.computeService.Instances.Insert(ctx.ProjectID, ctx.ZoneID, instance).Do()
+	if err != nil {
+		return "", fmt.Errorf("failed to create instance: %v", err)
+	}
+	if err := ctx.waitForCompletion("create", op.Name, false); err != nil {
+		return "", err
+	}
+
+	<-ctx.apiRateGate
+	inst, err := ctx.computeService.Instances.Get(ctx.ProjectID, ctx.ZoneID, name).Do()
+	if err != nil {
+		return "", fmt.Errorf("error getting instance %s details after creation: %v", name, err)
+	}
+
+	// Finds its internal IP.
+	ip := ""
+	for _, iface := range inst.NetworkInterfaces {
+		if strings.HasPrefix(iface.NetworkIP, "10.") {
+			ip = iface.NetworkIP
+			break
+		}
+	}
+	if ip == "" {
+		return "", fmt.Errorf("didn't find instance internal IP address")
+	}
+	return ip, nil
+}
+
+func (ctx *Context) DeleteInstance(name string) error {
+	<-ctx.apiRateGate
+	op, err := ctx.computeService.Instances.Delete(ctx.ProjectID, ctx.ZoneID, name).Do()
+	apiErr, ok := err.(*googleapi.Error)
+	if ok && apiErr.Code == 404 {
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("failed to delete instance: %v", err)
+	}
+	if err := ctx.waitForCompletion("delete", op.Name, true); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (ctx *Context) waitForCompletion(desc, opName string, ignoreNotFound bool) error {
+	for {
+		time.Sleep(2 * time.Second)
+		<-ctx.apiRateGate
+		op, err := ctx.computeService.ZoneOperations.Get(ctx.ProjectID, ctx.ZoneID, opName).Do()
+		if err != nil {
+			return fmt.Errorf("failed to get %v operation %v: %v", desc, opName, err)
+		}
+		switch op.Status {
+		case "PENDING", "RUNNING":
+			continue
+		case "DONE":
+			if op.Error != nil {
+				reason := ""
+				for _, operr := range op.Error.Errors {
+					if ignoreNotFound && operr.Code == "RESOURCE_NOT_FOUND" {
+						return nil
+					}
+					reason += fmt.Sprintf("%+v.", operr)
+				}
+				return fmt.Errorf("%v operation failed: %v", desc, reason)
+			}
+			return nil
+		default:
+			return fmt.Errorf("unknown %v operation status %q: %+v", desc, op.Status, op)
+		}
+	}
+}

syz-gce/syz-gce.go

@@ -0,0 +1,211 @@
+// Copyright 2016 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+// sudo apt-get install golang-go clang-format
+
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"cloud.google.com/go/storage"
+	"golang.org/x/net/context"
+	"google.golang.org/api/compute/v0.beta"
+)
+
+var (
+	flagConfig = flag.String("config", "", "config file")
+
+	cfg           *Config
+	ctx           context.Context
+	storageClient *storage.Client
+	computeService *compute.Service
+)
+
+type Config struct {
+	Image_Archive string
+	Image_Path    string
+	Http_Port     int
+	Machine_Type  string
+	Machine_Count int
+	Sandbox       string
+	Procs         int
+}
+
+func main() {
+	flag.Parse()
+	cfg = readConfig(*flagConfig)
+
+	var err error
+	ctx = context.Background()
+	storageClient, err = storage.NewClient(ctx)
+	if err != nil {
+		fatalf("failed to create cloud storage client: %v", err)
+	}
+
+	tokenSource, err := google.DefaultTokenSource(ctx, compute.CloudPlatformScope)
+	if err != nil {
+		fatalf("failed to get a token source: %v", err)
+	}
+	httpClient := oauth2.NewClient(ctx, tokenSource)
+	computeService, _ = compute.New(httpClient)
+
+	archive, updated, err := openFile(cfg.Image_Archive)
+	if err != nil {
+		fatalf("%v", err)
+	}
+	log.Printf("archive updated: %v", updated)
+
+	if false {
+		if err := os.RemoveAll("image"); err != nil {
+			fatalf("failed to remove image dir: %v", err)
+		}
+		if err := downloadAndExtract(archive, "image"); err != nil {
+			fatalf("failed to download and extract %v: %v", cfg.Image_Archive, err)
+		}
+
+	if err := uploadFile("image/disk.tar.gz", cfg.Image_Path); err != nil {
+		fatalf("failed to upload image: %v", err)
+	}
+	}
+
+
+		
+
+	if false {
+		syzBin, err := updateSyzkallerBuild()
+		if err != nil {
+			fatalf("failed to update/build syzkaller: %v", err)
+		}
+		_ = syzBin
+	}
+}
+
+func readConfig(filename string) *Config {
+	if filename == "" {
+		fatalf("supply config in -config flag")
+	}
+	data, err := ioutil.ReadFile(filename)
+	if err != nil {
+		fatalf("failed to read config file: %v", err)
+	}
+	cfg := new(Config)
+	if err := json.Unmarshal(data, cfg); err != nil {
+		fatalf("failed to parse config file: %v", err)
+	}
+	return cfg
+}
+
+func openFile(file string) (*storage.ObjectHandle, time.Time, error) {
+	pos := strings.IndexByte(file, '/')
+	if pos == -1 {
+		return nil, time.Time{}, fmt.Errorf("invalid GCS file name: %v", file)
+	}
+	bkt := storageClient.Bucket(file[:pos])
+	f := bkt.Object(file[pos+1:])
+	attrs, err := f.Attrs(ctx)
+	if err != nil {
+		return nil, time.Time{}, fmt.Errorf("failed to read %v attributes: %v", file, err)
+	}
+	if !attrs.Deleted.IsZero() {
+		return nil, time.Time{}, fmt.Errorf("file %v is deleted", file)
+	}
+	f = f.WithConditions(
+		storage.IfGenerationMatch(attrs.Generation),
+		storage.IfMetaGenerationMatch(attrs.MetaGeneration),
+	)
+	return f, attrs.Updated, nil
+}
+
+func downloadAndExtract(f *storage.ObjectHandle, dir string) error {
+	r, err := f.NewReader(ctx)
+	if err != nil {
+		return err
+	}
+	defer r.Close()
+	gz, err := gzip.NewReader(r)
+	if err != nil {
+		return err
+	}
+	ar := tar.NewReader(gz)
+	for {
+		hdr, err := ar.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+		log.Printf("extracting file: %v", hdr.Name)
+		if len(hdr.Name) == 0 || hdr.Name[len(hdr.Name)-1] == '/' {
+			continue
+		}
+		base, file := filepath.Split(hdr.Name)
+		if err := os.MkdirAll(filepath.Join(dir, base), 0700); err != nil {
+			return err
+		}
+		dst, err := os.OpenFile(filepath.Join(dir, base, file), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
+		if err != nil {
+			return err
+		}
+		_, err = io.Copy(dst, ar)
+		dst.Close()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func uploadFile(localFile string, gcsFile string) error {
+	local, err := os.Open(localFile)
+	if err != nil {
+		return err
+	}
+	defer local.Close()
+	pos := strings.IndexByte(gcsFile, '/')
+	if pos == -1 {
+		return fmt.Errorf("invalid GCS file name: %v", gcsFile)
+	}
+	bkt := storageClient.Bucket(gcsFile[:pos])
+	f := bkt.Object(gcsFile[pos+1:])
+	w := f.NewWriter(ctx)
+	defer w.Close()
+	io.Copy(w, local)
+	return nil
+}
+
+func updateSyzkallerBuild() (string, error) {
+	gopath, err := filepath.Abs("gopath")
+	if err != nil {
+		return "", err
+	}
+	goGet := exec.Command("go", "get", "-u", "-d", "github.com/google/syzkaller/syz-manager")
+	goGet.Env = append([]string{"GOPATH=" + gopath}, os.Environ()...)
+	if output, err := goGet.CombinedOutput(); err != nil {
+		return "", fmt.Errorf("%v\n%s", err, output)
+	}
+	makeCmd := exec.Command("make")
+	makeCmd.Env = append([]string{"GOPATH=" + gopath}, os.Environ()...)
+	makeCmd.Dir = "gopath/src/github.com/google/syzkaller"
+	if output, err := makeCmd.CombinedOutput(); err != nil {
+		return "", fmt.Errorf("%v\n%s", err, output)
+	}
+	return "gopath/src/github.com/google/syzkaller/bin", nil
+}
+
+func fatalf(msg string, args ...interface{}) {
+	log.Fatalf(msg, args...)
+}

syz-manager/manager.go

@@ -86,6 +86,7 @@ func main() {
 
 func RunManager(cfg *config.Config, syscalls map[int]bool, suppressions []*regexp.Regexp) {
 	crashdir := filepath.Join(cfg.Workdir, "crashes")
+	os.MkdirAll(crashdir)
 
 	enabledSyscalls := ""
 	if len(syscalls) != 0 {