Bug 1250709 - Taskcluster-based browser rooting hazard analysis, r=garndt

--HG--
extra : rebase_source : 62bb1300945125086ec35b42e44a33fb2eb60d7e
extra : source : b5bffc3757191411f15de9f8ed432f64fe69703c
extra : histedit_source : 17b355c3cd279a7e5c07f48ca8120c4703c146c6

browser/config/mozconfigs/linux64/hazards

@@ -1,5 +1,5 @@
-# This mozconfig is used when compiling the browser for the SM(Hf) rooting
-# hazard analysis build, see
+# This mozconfig is used when compiling the browser for the rooting hazard
+# analysis build (labeled H on treeherder). See
 # https://wiki.mozilla.org/Javascript:SpiderMonkey:ExactStackRooting
 
 # Do NOT include build/unix/mozconfig.linux because it points directly at the
@@ -21,12 +21,13 @@ mk_add_options MOZ_OBJDIR=obj-analyzed
 ac_add_options --enable-debug
 ac_add_options --enable-tests
 ac_add_options --enable-optimize
+ac_add_options --with-compiler-wrapper=$TOOLTOOL_DIR/sixgill/usr/libexec/sixgill/scripts/wrap_gcc/basecc
+ac_add_options --without-ccache
 
 CFLAGS="$CFLAGS -Wno-attributes"
 CPPFLAGS="$CPPFLAGS -Wno-attributes"
 CXXFLAGS="$CXXFLAGS -Wno-attributes"
 
-TOOLTOOL_DIR="$(dirname $topsrcdir)"
 export PKG_CONFIG_LIBDIR=/usr/lib64/pkgconfig:/usr/share/pkgconfig
 . $topsrcdir/build/unix/mozconfig.gtk
 

testing/taskcluster/scripts/builder/build-browser-haz-linux.sh

@@ -0,0 +1,29 @@
+#!/bin/bash -ex
+
+################################### build-mulet-haz-linux.sh ###################################
+# Ensure all the scripts in this dir are on the path....
+DIRNAME=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+PATH=$DIRNAME:$PATH
+WORKSPACE=$( cd "$1" && pwd )
+export TOOLTOOL_DIR="$WORKSPACE"
+
+. desktop-setup.sh
+. hazard-analysis.sh
+
+build_js_shell
+
+# Artifacts folder is outside of the cache.
+mkdir -p $HOME/artifacts/ || true
+
+function onexit () {
+    grab_artifacts "$WORKSPACE/analysis" "$HOME/artifacts"
+}
+
+trap onexit EXIT
+
+configure_analysis "$WORKSPACE/analysis"
+run_analysis "$WORKSPACE/analysis" browser
+
+check_hazards "$WORKSPACE/analysis"
+
+################################### script end ###################################

testing/taskcluster/scripts/builder/get-objdir.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 
 from __future__ import print_function
 import sys

testing/taskcluster/scripts/builder/hazard-analysis.sh

@@ -1,6 +1,10 @@
 #!/bin/bash -ex
 
-HAZARD_SHELL_OBJDIR=$MOZ_OBJDIR/obj-haz-shell
+[ -n "$WORKSPACE" ]
+[ -n "$MOZ_OBJDIR" ]
+[ -n "$GECKO_DIR" ]
+
+HAZARD_SHELL_OBJDIR=$WORKSPACE/obj-haz-shell
 JS_SRCDIR=$GECKO_DIR/js/src
 ANALYSIS_SRCDIR=$JS_SRCDIR/devtools/rootAnalysis
 
@@ -11,6 +15,12 @@ TOOLTOOL_MANIFEST=js/src/devtools/rootAnalysis/build/sixgill.manifest
 export CC="$TOOLTOOL_DIR/gcc/bin/gcc"
 export CXX="$TOOLTOOL_DIR/gcc/bin/g++"
 
+PYTHON=python2.7
+if ! which $PYTHON; then
+    PYTHON=python
+fi
+
+
 function build_js_shell () {
     ( cd $JS_SRCDIR; autoconf-2.13 )
     mkdir -p $HAZARD_SHELL_OBJDIR || true
@@ -45,7 +55,7 @@ function run_analysis () {
 
     (
         cd "$analysis_dir"
-        python "$ANALYSIS_SRCDIR/analyze.py" --buildcommand="$GECKO_DIR/testing/mozharness/scripts/spidermonkey/build.${build_type}"
+        $PYTHON "$ANALYSIS_SRCDIR/analyze.py" --buildcommand="$GECKO_DIR/testing/mozharness/scripts/spidermonkey/build.${build_type}"
     )
 }
 
@@ -59,13 +69,16 @@ function grab_artifacts () {
         cd "$analysis_dir"
         ls -lah
 
+        # Do not error out if no files found
+        shopt -s nullglob
+        set +e
         for f in *.txt *.lst; do
             gzip -9 -c "$f" > "${artifacts}/$f.gz"
         done
 
         # Check whether the user requested .xdb file upload in the top commit comment
 
-        if hg --cwd "$GECKO_DIR" log -l1 --template '{desc}\n' | grep -q 'haz: --upload-xdbs'; then
+        if hg --cwd "$GECKO_DIR" log -l1 --template '{desc}\n' | grep -q -- '--upload-xdbs'; then
             for f in *.xdb; do
                 bzip2 -c "$f" > "${artifacts}/$f.bz2"
             done
@@ -74,8 +87,19 @@ function grab_artifacts () {
 }
 
 function check_hazards () {
-    if grep 'Function.*has unrooted.*live across GC call' "$1"/rootingHazards.txt; then
-        echo "TEST-UNEXPECTED-FAIL hazards detected" >&2
+    (
+    set +e
+    NUM_HAZARDS=$(grep -c 'Function.*has unrooted.*live across GC call' "$1"/rootingHazards.txt)
+    NUM_UNSAFE=$(grep -c '^Function.*takes unsafe address of unrooted' "$1"/refs.txt)
+    NUM_UNNECESSARY=$(grep -c '^Function.* has unnecessary root' "$1"/unnecessary.txt)
+
+    echo "TinderboxPrint: $NUM_HAZARDS rooting hazards"
+    echo "TinderboxPrint: $NUM_UNSAFE unsafe references to unrooted GC pointers"
+    echo "TinderboxPrint: $NUM_UNSAFE unnecessary roots"
+
+    if [ $NUM_HAZARDS -gt 0 ]; then
+        echo "TEST-UNEXPECTED-FAIL $NUM_HAZARDS hazards detected" >&2
         exit 1
     fi
+    )
 }

testing/taskcluster/tasks/branches/try/job_flags.yml

@@ -75,6 +75,12 @@ builds:
     types:
       opt:
         task: tasks/builds/opt_linux64_st-an_clobber.yml
+  linux64-haz:
+    platforms:
+      - Linux64
+    types:
+      debug:
+        task: tasks/builds/haz_linux.yml
   macosx64:
     platforms:
       - MacOSX64

testing/taskcluster/tasks/builds/haz_linux.yml

@@ -0,0 +1,59 @@
+$inherits:
+  from: 'tasks/builds/firefox_base.yml'
+  variables:
+    build_name: 'browser-haz'
+    build_type: 'debug'
+task:
+  metadata:
+    name: '[TC] Browser Hazard Linux'
+    description: Browser Hazard Analysis Linux
+
+  workerType: dbg-linux64
+
+  scopes:
+    - 'docker-worker:cache:tooltool-cache'
+    - 'docker-worker:relengapi-proxy:tooltool.download.public'
+    - 'docker-worker:cache:level-{{level}}-{{project}}-build-linux64-haz-workspace'
+
+  payload:
+    cache:
+      level-{{level}}-{{project}}-build-linux64-haz-workspace: '/home/worker/workspace'
+      tooltool-cache: '/home/worker/tools/tooltool-cache'
+
+    features:
+      relengAPIProxy: true
+
+    env:
+      MOZCONFIG: 'browser/config/mozconfigs/linux64/hazards'
+      TOOLTOOL_CACHE: '/home/worker/tools/tooltool-cache'
+      TOOLTOOL_MANIFEST: 'gecko/browser/config/tooltool-manifests/linux64/releng.manifest'
+
+    maxRunTime: 36000
+
+    command:
+      - /bin/bash
+      - -c
+      - >
+        tc-vcs checkout workspace/gecko "$GECKO_BASE_REPOSITORY" "$GECKO_HEAD_REPOSITORY" "$GECKO_HEAD_REV" "$GECKO_HEAD_REF" &&
+        cd ./workspace/gecko/testing/taskcluster/scripts/builder &&
+        ./build-browser-haz-linux.sh $HOME/workspace
+
+  extra:
+    treeherderEnv:
+      - production
+      - staging
+    treeherder:
+      machine:
+        # see https://github.com/mozilla/treeherder/blob/master/ui/js/values.js
+        platform: linux64
+      groupSymbol: tc
+      groupName: Submitted by taskcluster for your pleasure
+      symbol: H
+      collection:
+        debug: true
+    # Rather then enforcing particular conventions we require that all build
+    # tasks provide the "build" extra field to specify where the build and tests
+    # files are located.
+    locations:
+      build: null
+      tests: null