sys/syz-extract: parallelize over files

sys/linux/extract.sh

@@ -56,13 +56,10 @@ generate_arch() {
 		echo "$OUT"
 		exit 1
 	fi
-	for F in $FILES; do
-		echo "extracting from $F"
-		bin/syz-extract -arch $1 -linux "$LINUX" -linuxbld "$LINUXBLD" "sys/linux/$F"
-		if [ $? -ne 0 ]; then
-			exit 1
-		fi
-	done
+	(cd sys/linux; ../../bin/syz-extract -arch $1 -linux "$LINUX" -linuxbld "$LINUXBLD" $FILES)
+	if [ $? -ne 0 ]; then
+		exit 1
+	fi
 	echo
 }
 

sys/syz-extract/extract.go

@@ -4,12 +4,15 @@
 package main
 
 import (
+	"bytes"
 	"flag"
 	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
+	"sync"
 
 	"github.com/google/syzkaller/pkg/ast"
 	"github.com/google/syzkaller/pkg/compiler"
@@ -20,7 +23,6 @@ var (
 	flagLinux    = flag.String("linux", "", "path to linux kernel source checkout")
 	flagLinuxBld = flag.String("linuxbld", "", "path to linux kernel build directory")
 	flagArch     = flag.String("arch", "", "arch to generate")
-	flagV        = flag.Int("v", 0, "verbosity")
 )
 
 type Arch struct {
@@ -38,13 +40,23 @@ var archs = map[string]*Arch{
 	"ppc64le": {[]string{"__ppc64__", "__PPC64__", "__powerpc64__"}, "powerpc", "asm/unistd.h", []string{"-D__powerpc64__"}},
 }
 
+type File struct {
+	name       string
+	undeclared map[string]bool
+	err        error
+}
+
 func main() {
+	failf := func(msg string, args ...interface{}) {
+		fmt.Fprintf(os.Stderr, msg+"\n", args...)
+		os.Exit(1)
+	}
+
 	flag.Parse()
 	if *flagLinux == "" {
 		failf("provide path to linux kernel checkout via -linux flag (or make extract LINUX= flag)")
 	}
 	if *flagLinuxBld == "" {
-		logf(1, "No kernel build directory provided, assuming in-place build")
 		*flagLinuxBld = *flagLinux
 	}
 	if *flagArch == "" {
@@ -53,54 +65,73 @@ func main() {
 	if archs[*flagArch] == nil {
 		failf("unknown arch %v", *flagArch)
 	}
-	if len(flag.Args()) != 1 {
-		failf("usage: syz-extract -linux=/linux/checkout -arch=arch sys/input_file.txt")
+	n := len(flag.Args())
+	if n == 0 {
+		failf("usage: syz-extract -linux=/linux/checkout -arch=arch input_file.txt...")
 	}
 
-	inname := flag.Args()[0]
-	outname := strings.TrimSuffix(inname, ".txt") + "_" + *flagArch + ".const"
+	files := make([]File, n)
+	inc := make(chan *File, n)
+	for i, f := range flag.Args() {
+		files[i].name = f
+		inc <- &files[i]
+	}
+	close(inc)
 
+	procs := runtime.GOMAXPROCS(0)
+	var wg sync.WaitGroup
+	wg.Add(procs)
+	for p := 0; p < procs; p++ {
+		go func() {
+			defer wg.Done()
+			for f := range inc {
+				f.undeclared, f.err = processFile(f.name)
+			}
+		}()
+	}
+	wg.Wait()
+	for _, f := range files {
+		fmt.Printf("extracting from %v\n", f.name)
+		if f.err != nil {
+			failf("%v", f.err)
+		}
+		for c := range f.undeclared {
+			fmt.Printf("undefined const: %v\n", c)
+		}
+	}
+}
+
+func processFile(inname string) (map[string]bool, error) {
+	outname := strings.TrimSuffix(inname, ".txt") + "_" + *flagArch + ".const"
 	indata, err := ioutil.ReadFile(inname)
 	if err != nil {
-		failf("failed to read input file: %v", err)
+		return nil, fmt.Errorf("failed to read input file: %v", err)
 	}
-
-	desc := ast.Parse(indata, filepath.Base(inname), nil)
+	errBuf := new(bytes.Buffer)
+	eh := func(pos ast.Pos, msg string) {
+		fmt.Fprintf(errBuf, "%v: %v\n", pos, msg)
+	}
+	desc := ast.Parse(indata, filepath.Base(inname), eh)
 	if desc == nil {
-		os.Exit(1)
+		return nil, fmt.Errorf("%v", errBuf.String())
 	}
-
-	consts := compileConsts(archs[*flagArch], desc)
-
-	data := compiler.SerializeConsts(consts)
-	if err := osutil.WriteFile(outname, data); err != nil {
-		failf("failed to write output file: %v", err)
-	}
-}
-
-func compileConsts(arch *Arch, desc *ast.Description) map[string]uint64 {
-	info := compiler.ExtractConsts(desc, nil)
+	info := compiler.ExtractConsts(desc, eh)
 	if info == nil {
-		os.Exit(1)
+		return nil, fmt.Errorf("%v", errBuf.String())
 	}
 	if len(info.Consts) == 0 {
-		return nil
+		return nil, nil
 	}
-	consts, err := fetchValues(arch.KernelHeaderArch, info.Consts,
-		append(info.Includes, arch.KernelInclude), info.Incdirs, info.Defines, arch.CFlags)
+	arch := archs[*flagArch]
+	includes := append(info.Includes, arch.KernelInclude)
+	consts, undeclared, err := fetchValues(arch.KernelHeaderArch, info.Consts,
+		includes, info.Incdirs, arch.CFlags, info.Defines)
 	if err != nil {
-		failf("%v", err)
+		return nil, err
 	}
-	return consts
-}
-
-func failf(msg string, args ...interface{}) {
-	fmt.Fprintf(os.Stderr, msg+"\n", args...)
-	os.Exit(1)
-}
-
-func logf(v int, msg string, args ...interface{}) {
-	if *flagV >= v {
-		fmt.Fprintf(os.Stderr, msg+"\n", args...)
+	data := compiler.SerializeConsts(consts)
+	if err := osutil.WriteFile(outname, data); err != nil {
+		return nil, fmt.Errorf("failed to write output file: %v", err)
 	}
+	return undeclared, nil
 }

sys/syz-extract/fetch.go

@@ -16,10 +16,11 @@ import (
 // fetchValues converts literal constants (e.g. O_APPEND) or any other C expressions
 // into their respective numeric values. It does so by builting and executing a C program
 // that prints values of the provided expressions.
-func fetchValues(arch string, vals []string, includes []string, incdirs []string, defines map[string]string, cflags []string) (map[string]uint64, error) {
-	bin, out, err := runCompiler(arch, nil, includes, incdirs, nil, cflags, nil)
+func fetchValues(arch string, vals, includes, incdirs, cflags []string,
+	defines map[string]string) (map[string]uint64, map[string]bool, error) {
+	bin, out, err := runCompiler(arch, nil, includes, incdirs, cflags, nil, nil)
 	if err != nil {
-		return nil, fmt.Errorf("failed to run gcc: %v\n%v", err, string(out))
+		return nil, nil, fmt.Errorf("failed to run gcc: %v\n%v", err, string(out))
 	}
 	os.Remove(bin)
 
@@ -29,7 +30,7 @@ func fetchValues(arch string, vals []string, includes []string, incdirs []string
 	}
 
 	undeclared := make(map[string]bool)
-	bin, out, err = runCompiler(arch, vals, includes, incdirs, defines, cflags, undeclared)
+	bin, out, err = runCompiler(arch, vals, includes, incdirs, cflags, defines, undeclared)
 	if err != nil {
 		for _, errMsg := range []string{
 			"error: ‘([a-zA-Z0-9_]+)’ undeclared",
@@ -40,21 +41,20 @@ func fetchValues(arch string, vals []string, includes []string, incdirs []string
 			for _, match := range matches {
 				val := string(match[1])
 				if !undeclared[val] && valMap[val] {
-					logf(0, "undefined const: %v", val)
 					undeclared[val] = true
 				}
 			}
 		}
-		bin, out, err = runCompiler(arch, vals, includes, incdirs, defines, cflags, undeclared)
+		bin, out, err = runCompiler(arch, vals, includes, incdirs, cflags, defines, undeclared)
 		if err != nil {
-			return nil, fmt.Errorf("failed to run gcc: %v\n%v", err, string(out))
+			return nil, nil, fmt.Errorf("failed to run gcc: %v\n%v", err, string(out))
 		}
 	}
 	defer os.Remove(bin)
 
 	out, err = exec.Command(bin).CombinedOutput()
 	if err != nil {
-		return nil, fmt.Errorf("failed to run flags binary: %v\n%v", err, string(out))
+		return nil, nil, fmt.Errorf("failed to run flags binary: %v\n%v", err, string(out))
 	}
 
 	flagVals := strings.Split(string(out), " ")
@@ -62,7 +62,7 @@ func fetchValues(arch string, vals []string, includes []string, incdirs []string
 		flagVals = nil
 	}
 	if len(flagVals) != len(vals)-len(undeclared) {
-		failf("fetched wrong number of values %v != %v - %v\nflagVals: %q\nvals: %q\nundeclared: %q",
+		return nil, nil, fmt.Errorf("fetched wrong number of values %v != %v - %v\nflagVals: %q\nvals: %q\nundeclared: %q",
 			len(flagVals), len(vals), len(undeclared),
 			flagVals, vals, undeclared)
 	}
@@ -77,14 +77,14 @@ func fetchValues(arch string, vals []string, includes []string, incdirs []string
 		}
 		n, err := strconv.ParseUint(v, 10, 64)
 		if err != nil {
-			failf("failed to parse value: %v (%v)", err, v)
+			return nil, nil, fmt.Errorf("failed to parse value: %v (%v)", err, v)
 		}
 		res[name] = n
 	}
-	return res, nil
+	return res, undeclared, nil
 }
 
-func runCompiler(arch string, vals []string, includes []string, incdirs []string, defines map[string]string, cflags []string, undeclared map[string]bool) (bin string, out []byte, err error) {
+func runCompiler(arch string, vals, includes, incdirs, cflags []string, defines map[string]string, undeclared map[string]bool) (bin string, out []byte, err error) {
 	includeText := ""
 	for _, inc := range includes {
 		includeText += fmt.Sprintf("#include <%v>\n", inc)