syz-ci: upload coverage reports to GCS

Upload coverage reports from all managers to GCS
after 6 hours of runtime (to make it more apples-to-apples).

pkg/gcs/gcs.go

@@ -107,6 +107,19 @@ func (client *Client) FileWriter(gcsFile string) (io.WriteCloser, error) {
 	return w, nil
 }
 
+// Publish lets any user read gcsFile.
+func (client *Client) Publish(gcsFile string) error {
+	bucket, filename, err := split(gcsFile)
+	if err != nil {
+		return err
+	}
+	obj := client.client.Bucket(bucket).Object(filename)
+	return obj.ACL().Set(client.ctx, storage.AllUsers, storage.RoleReader)
+}
+
+// Where things get published.
+const PublicPrefix = "https://storage.googleapis.com/"
+
 func split(file string) (bucket, filename string, err error) {
 	pos := strings.IndexByte(file, '/')
 	if pos == -1 {

syz-ci/manager.go

@@ -5,7 +5,9 @@ package main
 
 import (
 	"fmt"
+	"io"
 	"io/ioutil"
+	"net/http"
 	"os"
 	"path/filepath"
 	"time"
@@ -13,6 +15,7 @@ import (
 	"github.com/google/syzkaller/dashboard/dashapi"
 	"github.com/google/syzkaller/pkg/build"
 	"github.com/google/syzkaller/pkg/config"
+	"github.com/google/syzkaller/pkg/gcs"
 	"github.com/google/syzkaller/pkg/hash"
 	"github.com/google/syzkaller/pkg/instance"
 	"github.com/google/syzkaller/pkg/log"
@@ -152,7 +155,7 @@ var kernelBuildSem = make(chan struct{}, 1)
 func (mgr *Manager) loop() {
 	lastCommit := ""
 	nextBuildTime := time.Now()
-	var managerRestartTime time.Time
+	var managerRestartTime, coverUploadTime time.Time
 	latestInfo := mgr.checkLatest()
 	if latestInfo != nil && time.Since(latestInfo.Time) < kernelRebuildPeriod/2 {
 		// If we have a reasonably fresh build,
@@ -204,6 +207,12 @@ loop:
 			}
 			nextBuildTime = time.Now().Add(rebuildAfter)
 		}
+		if !coverUploadTime.IsZero() && time.Now().After(coverUploadTime) {
+			coverUploadTime = time.Time{}
+			if err := mgr.uploadCoverReport(); err != nil {
+				mgr.Errorf("failed to upload cover report: %v", err)
+			}
+		}
 
 		select {
 		case <-mgr.stop:
@@ -214,6 +223,9 @@ loop:
 		if latestInfo != nil && (latestInfo.Time != managerRestartTime || mgr.cmd == nil) {
 			managerRestartTime = latestInfo.Time
 			mgr.restartManager()
+			if mgr.cmd != nil && mgr.managercfg.Cover && mgr.cfg.CoverUploadPath != "" {
+				coverUploadTime = time.Now().Add(6 * time.Hour)
+			}
 		}
 
 		select {
@@ -581,6 +593,32 @@ func (mgr *Manager) pollCommits(buildCommit string) ([]string, []dashapi.FixComm
 	return present, fixCommits, nil
 }
 
+func (mgr *Manager) uploadCoverReport() error {
+	GCS, err := gcs.NewClient()
+	if err != nil {
+		return fmt.Errorf("failed to create GCS client: %v", err)
+	}
+	defer GCS.Close()
+	resp, err := http.Get(fmt.Sprintf("http://%v/cover", mgr.managercfg.HTTP))
+	if err != nil {
+		return fmt.Errorf("failed to get report: %v", err)
+	}
+	defer resp.Body.Close()
+	gcsPath := filepath.Join(mgr.cfg.CoverUploadPath, mgr.name+".html")
+	gcsWriter, err := GCS.FileWriter(gcsPath)
+	if err != nil {
+		return fmt.Errorf("failed to create GCS writer: %v", err)
+	}
+	if _, err := io.Copy(gcsWriter, resp.Body); err != nil {
+		gcsWriter.Close()
+		return fmt.Errorf("failed to copy report: %v", err)
+	}
+	if err := gcsWriter.Close(); err != nil {
+		return fmt.Errorf("failed to close gcs writer: %v", err)
+	}
+	return GCS.Publish(gcsPath)
+}
+
 // Errorf logs non-fatal error and sends it to dashboard.
 func (mgr *Manager) Errorf(msg string, args ...interface{}) {
 	log.Logf(0, mgr.name+": "+msg, args...)

syz-ci/syz-ci.go

@@ -85,6 +85,8 @@ type Config struct {
 	SyzkallerBranch string `json:"syzkaller_branch"`
 	// Dir with additional syscall descriptions (.txt and .const files).
 	SyzkallerDescriptions string `json:"syzkaller_descriptions"`
+	// GCS path to upload coverage reports from managers (optional).
+	CoverUploadPath string `json:"cover_upload_path"`
 	// Enable patch testing jobs.
 	EnableJobs bool             `json:"enable_jobs"`
 	Managers   []*ManagerConfig `json:"managers"`
@@ -163,6 +165,17 @@ func main() {
 		}()
 	}
 
+	// For testing. Racy. Use with care.
+	http.HandleFunc("/upload_cover", func(w http.ResponseWriter, r *http.Request) {
+		for _, mgr := range managers {
+			if err := mgr.uploadCoverReport(); err != nil {
+				w.Write([]byte(fmt.Sprintf("failed for %v: %v <br>\n", mgr.name, err)))
+				return
+			}
+			w.Write([]byte(fmt.Sprintf("upload cover for %v <br>\n", mgr.name)))
+		}
+	})
+
 	wg.Wait()
 
 	select {