Bug 1280637 - Implement tsan (thread sanitizer), asan (address sanitizer), and msan (uninitialized memory) jobs, r=terrence

MozReview-Commit-ID: JuFVqAb9C4s

--HG--
rename : browser/config/tooltool-manifests/linux64/asan.manifest => browser/config/tooltool-manifests/linux64/msan.manifest
extra : rebase_source : 1a5e77992388b0323bb639ee2fd4285a68b94505

browser/config/tooltool-manifests/linux64/msan.manifest

@@ -0,0 +1,26 @@
+[
+{
+"version": "gcc 4.8.5 + PR64905",
+"size": 80160264,
+"digest": "c1a9dc9da289b8528874d16300b9d13a997cec99195bb0bc46ff665216d8535d6d6cb5af6b4b1f2749af6815dab12e703fdb3849014e5c23a70eff351a0baf4e",
+"algorithm": "sha512",
+"filename": "gcc.tar.xz",
+"unpack": true
+},
+{
+"version": "clang 3.8.0, libgcc 4.8.5",
+"size": 139183100,
+"digest": "a056a151d4f25f415b6d905136c3fa8d51d12a5a815c3df37d5663c67d59571736641a4c990884a69f78ea6b5e37a6a7bfff0417dfe38936d842d6fa0776ae54",
+"algorithm": "sha512",
+"filename": "clang.tar.xz",
+"unpack": true
+},
+{
+"size": 12072532,
+"digest": "3915f8ec396c56a8a92e6f9695b70f09ce9d1582359d1258e37e3fd43a143bc974410e4cfc27f500e095f34a8956206e0ebf799b7287f0f38def0d5e34ed71c9",
+"algorithm": "sha512",
+"filename": "gtk3.tar.xz",
+"setup": "setup.sh",
+"unpack": true
+}
+]

js/src/Makefile.in

@@ -75,7 +75,10 @@ ifdef MOZ_ASAN
 JITTEST_SANITIZER_ENV=ASAN_SYMBOLIZER_PATH='$(LLVM_SYMBOLIZER)'
 endif
 ifdef MOZ_TSAN
-JITTEST_SANITIZER_ENV=TSAN_OPTIONS='external_symbolizer_path=$(LLVM_SYMBOLIZER)'
+JITTEST_SANITIZER_ENV=TSAN_OPTIONS="external_symbolizer_path=$(LLVM_SYMBOLIZER) $$TSAN_OPTIONS"
+endif
+ifdef MOZ_MSAN
+JITTEST_SANITIZER_ENV=MSAN_SYMBOLIZER_PATH='$(LLVM_SYMBOLIZER)'
 endif
 endif
 

js/src/devtools/automation/autospider.py

@@ -11,19 +11,21 @@ import shutil
 import subprocess
 import sys
 
-from collections import namedtuple
+from collections import Counter, namedtuple
 from os import environ as env
 from subprocess import Popen
 from threading import Timer
 
-Dirs = namedtuple('Dirs', ['scripts', 'js_src', 'source'])
+Dirs = namedtuple('Dirs', ['scripts', 'js_src', 'source', 'tooltool'])
 
 
 def directories(pathmodule, cwd, fixup=lambda s: s):
     scripts = pathmodule.join(fixup(cwd), fixup(pathmodule.dirname(__file__)))
     js_src = pathmodule.abspath(pathmodule.join(scripts, "..", ".."))
     source = pathmodule.abspath(pathmodule.join(js_src, "..", ".."))
-    return Dirs(scripts, js_src, source)
+    tooltool = pathmodule.abspath(env.get('TOOLTOOL_CHECKOUT',
+                                          pathmodule.join(source, "..")))
+    return Dirs(scripts, js_src, source, tooltool)
 
 # Some scripts will be called with sh, which cannot use backslashed
 # paths. So for direct subprocess.* invocation, use normal paths from
@@ -98,7 +100,6 @@ def set_vars_from_script(script, vars):
         for var in vars:
             if var in originals and len(originals[var]) > 0:
                 env[var] = "%s;%s" % (env[var], originals[var])
-                print("orig appended, %s = %s" % (var, env[var]))
 
 
 def call_alternates(binaries, command_args, *args, **kwargs):
@@ -138,6 +139,7 @@ if call_alternates(autoconfs, [], cwd=DIR.js_src) != 0:
     sys.exit(1)
 
 OBJDIR = os.path.join(DIR.source, args.objdir)
+OUTDIR = os.path.join(OBJDIR, "out")
 POBJDIR = posixpath.join(PDIR.source, args.objdir)
 AUTOMATION = env.get('AUTOMATION', False)
 MAKE = env.get('MAKE', 'make')
@@ -145,6 +147,11 @@ MAKEFLAGS = '-j6'
 CONFIGURE_ARGS = variant['configure-args']
 UNAME_M = subprocess.check_output(['uname', '-m']).strip()
 
+# Any jobs that wish to produce additional output can save them into the upload
+# directory if there is such a thing, falling back to OBJDIR.
+env.setdefault('MOZ_UPLOAD_DIR', OBJDIR)
+ensure_dir_exists(env['MOZ_UPLOAD_DIR'], clobber=False)
+
 # Some of the variants request a particular word size (eg ARM simulators).
 word_bits = variant.get('bits')
 
@@ -161,17 +168,30 @@ if word_bits is None and args.platform:
 if word_bits is None:
     word_bits = 64 if UNAME_M == 'x86_64' else 32
 
+if 'compiler' in variant:
+    compiler = variant['compiler']
+elif platform.system() == 'Darwin':
+    compiler = 'clang'
+elif platform.system() == 'Windows':
+    compiler = 'cl'
+else:
+    compiler = 'gcc'
+
+cxx = {'clang': 'clang++', 'gcc': 'g++', 'cl': 'cl'}.get(compiler)
+
+compiler_dir = env.get('GCCDIR', os.path.join(DIR.tooltool, compiler))
+if os.path.exists(os.path.join(compiler_dir, 'bin', compiler)):
+    env.setdefault('CC', os.path.join(compiler_dir, 'bin', compiler))
+    env.setdefault('CXX', os.path.join(compiler_dir, 'bin', cxx))
+    platlib = 'lib64' if word_bits == 64 else 'lib'
+    env.setdefault('LD_LIBRARY_PATH', os.path.join(compiler_dir, platlib))
+else:
+    env.setdefault('CC', compiler)
+    env.setdefault('CXX', cxx)
+
 if platform.system() == 'Darwin':
     set_vars_from_script(os.path.join(DIR.scripts, 'macbuildenv.sh'),
                          ['CC', 'CXX'])
-elif platform.system() == 'Linux':
-    if AUTOMATION:
-        GCCDIR = env.get('GCCDIR', os.path.join(DIR.source, '..', 'gcc'))
-        CONFIGURE_ARGS += ' --with-ccache'
-        env.setdefault('CC', os.path.join(GCCDIR, 'bin', 'gcc'))
-        env.setdefault('CXX', os.path.join(GCCDIR, 'bin', 'g++'))
-        platlib = 'lib64' if word_bits == 64 else 'lib'
-        env.setdefault('LD_LIBRARY_PATH', os.path.join(GCCDIR, platlib))
 elif platform.system() == 'Windows':
     MAKE = env.get('MAKE', 'mozmake')
     os.environ['SOURCE'] = DIR.source
@@ -180,25 +200,26 @@ elif platform.system() == 'Windows':
     set_vars_from_script(posixpath.join(PDIR.scripts, 'winbuildenv.sh'),
                          ['PATH', 'INCLUDE', 'LIB', 'LIBPATH', 'CC', 'CXX'])
 
-if word_bits == 64:
-    if platform.system() == 'Windows':
-        CONFIGURE_ARGS += ' --target=x86_64-pc-mingw32 --host=x86_64-pc-mingw32'
-else:
-    if platform.system() == 'Darwin':
+# Compiler flags, based on word length
+if word_bits == 32:
+    if compiler == 'clang':
         env['CC'] = '{CC} -arch i386'.format(**env)
         env['CXX'] = '{CXX} -arch i386'.format(**env)
-    elif platform.system() == 'Windows':
-        CONFIGURE_ARGS += ' --target=i686-pc-mingw32 --host=i686-pc-mingw32'
-    else:
-        env.setdefault('CC', 'gcc')
-        env.setdefault('CXX', 'g++')
+    elif compiler == 'gcc':
         env['CC'] = '{CC} -m32'.format(**env)
         env['CXX'] = '{CXX} -m32'.format(**env)
         env['AR'] = 'ar'
 
-    if platform.system() == 'Linux':
-        if AUTOMATION and UNAME_M != 'arm':
+# Configure flags, based on word length and cross-compilation
+if word_bits == 32:
+    if platform.system() == 'Windows':
+        CONFIGURE_ARGS += ' --target=i686-pc-mingw32 --host=i686-pc-mingw32'
+    elif platform.system() == 'Linux':
+        if UNAME_M != 'arm':
             CONFIGURE_ARGS += ' --target=i686-pc-linux --host=i686-pc-linux'
+else:
+    if platform.system() == 'Windows':
+        CONFIGURE_ARGS += ' --target=x86_64-pc-mingw32 --host=x86_64-pc-mingw32'
 
 # Timeouts.
 ACTIVE_PROCESSES = set()
@@ -214,6 +235,7 @@ timer.daemon = True
 timer.start()
 
 ensure_dir_exists(OBJDIR, clobber=not args.dep)
+ensure_dir_exists(OUTDIR)
 
 
 def run_command(command, check=False, **kwargs):
@@ -229,6 +251,16 @@ def run_command(command, check=False, **kwargs):
         raise subprocess.CalledProcessError(status, command, output=stderr)
     return stdout, stderr, status
 
+# Add in environment variable settings for this variant. Normally used to
+# modify the flags passed to the shell or to set the GC zeal mode.
+for k, v in variant.get('env', {}).items():
+    env[k] = v.format(
+        DIR=DIR.scripts,
+        TOOLTOOL_CHECKOUT=DIR.tooltool,
+        MOZ_UPLOAD_DIR=env['MOZ_UPLOAD_DIR'],
+        OUTDIR=OUTDIR,
+    )
+
 CONFIGURE_ARGS += ' --enable-nspr-build'
 CONFIGURE_ARGS += ' --prefix={OBJDIR}/dist'.format(OBJDIR=POBJDIR)
 run_command(['sh', '-c', posixpath.join(PDIR.js_src, 'configure') + ' ' + CONFIGURE_ARGS], check=True)
@@ -247,11 +279,6 @@ def run_test_command(command, **kwargs):
 
 test_suites = set(['jstests', 'jittest', 'jsapitests', 'checks'])
 
-# Add in environment variable settings for this variant. Normally used to
-# modify the flags passed to the shell or to set the GC zeal mode.
-for k, v in variant.get('env', {}).items():
-    env[k] = v.format(DIR=DIR.scripts)
-
 # Need a platform name to use as a key in variant files.
 if args.platform:
     variant_platform = args.platform.split("-")[0]
@@ -294,6 +321,36 @@ if 'jsapitests' in test_suites:
 if 'jstests' in test_suites:
     results.append(run_test_command([MAKE, 'check-jstests']))
 
+if args.variant in ('tsan', 'msan'):
+    files = filter(lambda f: f.startswith("sanitize_log."), os.listdir(OUTDIR))
+    fullfiles = [os.path.join(OUTDIR, f) for f in files]
+
+    # Summarize results
+    sites = Counter()
+    for filename in fullfiles:
+        with open(os.path.join(OUTDIR, filename), 'rb') as fh:
+            for line in fh:
+                m = re.match(r'^SUMMARY: \w+Sanitizer: (?:data race|use-of-uninitialized-value) (.*)',
+                             line.strip())
+                if m:
+                    sites[m.group(1)] += 1
+
+    # Write a summary file and display it to stdout.
+    summary_filename = os.path.join(env['MOZ_UPLOAD_DIR'], "%s_summary.txt" % args.variant)
+    with open(summary_filename, 'wb') as outfh:
+        for location, count in sites.most_common():
+            print >> outfh, "%d %s" % (count, location)
+    print(open(summary_filename, 'rb').read())
+
+    # Gather individual results into a tarball. Note that these are
+    # distinguished only by pid of the JS process running within each test, so
+    # given the 16-bit limitation of pids, it's totally possible that some of
+    # these files will be lost due to being overwritten.
+    command = ['tar', '-C', OUTDIR, '-zcf',
+               os.path.join(env['MOZ_UPLOAD_DIR'], '%s.tar.gz' % args.variant)]
+    command += files
+    subprocess.call(command)
+
 for st in results:
     if st != 0:
         sys.exit(st)

js/src/devtools/automation/variants/asan

@@ -0,0 +1,7 @@
+{
+    "configure-args": "--disable-debug --enable-optimize --enable-debug-symbols='-gline-tables-only' --disable-jemalloc --enable-address-sanitizer",
+    "compiler": "clang",
+    "env": {
+        "LLVM_SYMBOLIZER": "{TOOLTOOL_CHECKOUT}/clang/bin/llvm-symbolizer"
+    }
+}

js/src/devtools/automation/variants/msan

@@ -0,0 +1,10 @@
+{
+    "configure-args": "--disable-debug --enable-optimize --enable-debug-symbols='-gline-tables-only' --disable-jemalloc --enable-memory-sanitizer",
+    "compiler": "clang",
+    "env": {
+        "LLVM_SYMBOLIZER": "{TOOLTOOL_CHECKOUT}/clang/bin/llvm-symbolizer",
+        "JITTEST_EXTRA_ARGS": "--jitflags=debug --ignore-timeouts={DIR}/cgc-jittest-timeouts.txt",
+        "JSTESTS_EXTRA_ARGS": "--exclude-file={DIR}/cgc-jstests-slow.txt",
+        "MSAN_OPTIONS": "log_path={OUTDIR}/sanitize_log"
+    }
+}

js/src/devtools/automation/variants/tsan

@@ -0,0 +1,10 @@
+{
+    "configure-args": "--disable-debug --enable-optimize --enable-debug-symbols='-gline-tables-only' --disable-jemalloc --enable-thread-sanitizer",
+    "compiler": "clang",
+    "env": {
+        "LLVM_SYMBOLIZER": "{TOOLTOOL_CHECKOUT}/clang/bin/llvm-symbolizer",
+        "JITTEST_EXTRA_ARGS": "--jitflags=debug --ignore-timeouts={DIR}/cgc-jittest-timeouts.txt",
+        "JSTESTS_EXTRA_ARGS": "--exclude-file={DIR}/cgc-jstests-slow.txt",
+        "TSAN_OPTIONS": "log_path={OUTDIR}/sanitize_log"
+    }
+}

taskcluster/ci/legacy/tasks/branches/base_jobs.yml

@@ -50,6 +50,9 @@ builds:
       - sm-compacting
       - sm-rootanalysis
       - sm-package
+      - sm-tsan
+      - sm-asan
+      - sm-msan
     types:
       opt:
         task: tasks/builds/opt_linux64.yml
@@ -129,6 +132,36 @@ builds:
       file_patterns:
        - js/public/**
        - js/src/**
+  sm-tsan:
+    platforms:
+      - Linux64
+    types:
+      opt:
+        task: tasks/builds/sm_tsan.yml
+    when:
+      file_patterns:
+       - js/public/**
+       - js/src/**
+  sm-asan:
+    platforms:
+      - Linux64
+    types:
+      opt:
+        task: tasks/builds/sm_asan.yml
+    when:
+      file_patterns:
+       - js/public/**
+       - js/src/**
+  sm-msan:
+    platforms:
+      - Linux64
+    types:
+      opt:
+        task: tasks/builds/sm_msan.yml
+    when:
+      file_patterns:
+       - js/public/**
+       - js/src/**
   sm-nonunified:
     platforms:
       - Linux64

taskcluster/ci/legacy/tasks/builds/sm_asan.yml

@@ -0,0 +1,16 @@
+$inherits:
+  from: 'tasks/builds/sm_variant_base.yml'
+  variables:
+    build_name: 'sm-asan'
+    build_type: 'opt'
+task:
+  payload:
+    env:
+      SPIDERMONKEY_VARIANT: 'asan'
+      TOOLTOOL_MANIFEST: 'browser/config/tooltool-manifests/linux64/asan.manifest'
+  metadata:
+    name: '[TC] Spidermonkey Address Sanitizer'
+    description: 'Spidermonkey Address Sanitizer'
+  extra:
+    treeherder:
+      symbol: asan

taskcluster/ci/legacy/tasks/builds/sm_msan.yml

@@ -0,0 +1,16 @@
+$inherits:
+  from: 'tasks/builds/sm_variant_base.yml'
+  variables:
+    build_name: 'sm-msan'
+    build_type: 'opt'
+task:
+  payload:
+    env:
+      SPIDERMONKEY_VARIANT: 'msan'
+      TOOLTOOL_MANIFEST: 'browser/config/tooltool-manifests/linux64/msan.manifest'
+  metadata:
+    name: '[TC] Spidermonkey Memory Sanitizer'
+    description: 'Spidermonkey Memory Sanitizer'
+  extra:
+    treeherder:
+      symbol: msan

taskcluster/ci/legacy/tasks/builds/sm_tsan.yml

@@ -0,0 +1,16 @@
+$inherits:
+  from: 'tasks/builds/sm_variant_base.yml'
+  variables:
+    build_name: 'sm-tsan'
+    build_type: 'opt'
+task:
+  payload:
+    env:
+      SPIDERMONKEY_VARIANT: 'tsan'
+      TOOLTOOL_MANIFEST: 'browser/config/tooltool-manifests/linux64/tsan.manifest'
+  metadata:
+    name: '[TC] Spidermonkey Thread Sanitizer'
+    description: 'Spidermonkey Thread Sanitizer'
+  extra:
+    treeherder:
+      symbol: tsan

taskcluster/scripts/builder/build-sm.sh

@@ -7,6 +7,7 @@ source $(dirname $0)/sm-tooltool-config.sh
 : ${PYTHON:=python2.7}
 
 # Run the script
+export MOZ_UPLOAD_DIR="$UPLOAD_DIR"
 AUTOMATION=1 $PYTHON $SRCDIR/js/src/devtools/automation/autospider.py $SPIDERMONKEY_VARIANT
 BUILD_STATUS=$?
 

taskcluster/scripts/builder/sm-tooltool-config.sh

@@ -44,18 +44,10 @@ esac
 # necessary for the JS shell, but it's less duplication to share tooltool
 # manifests.
 BROWSER_PLATFORM=$PLATFORM_OS$BITS
-TOOLTOOL_MANIFEST="$SRCDIR/browser/config/tooltool-manifests/$BROWSER_PLATFORM/releng.manifest"
+: ${TOOLTOOL_MANIFEST:=browser/config/tooltool-manifests/$BROWSER_PLATFORM/releng.manifest}
 
-tc-vcs checkout $WORK/tooltool $TOOLTOOL_REPO $TOOLTOOL_REPO $TOOLTOOL_REV
-(cd $WORK && python tooltool/tooltool.py --url $TOOLTOOL_SERVER -m $TOOLTOOL_MANIFEST fetch ${TOOLTOOL_CACHE:+ -c $TOOLTOOL_CACHE})
+: ${TOOLTOOL_CHECKOUT:=$WORK}
+export TOOLTOOL_CHECKOUT
 
-# Point to the appropriate compiler, assuming taskcluster will one day run on non-Linux OSes.
-case "$PLATFORM_OS" in
-    linux)
-        export PATH="$WORK/gcc/bin":$PATH
-        export GCCDIR="$WORK/gcc"
-        ;;
-    macosx)
-        export PATH="$WORK/clang/bin":$PATH
-        ;;
-esac
+tc-vcs checkout $TOOLTOOL_CHECKOUT/tooltool $TOOLTOOL_REPO $TOOLTOOL_REPO $TOOLTOOL_REV
+(cd $TOOLTOOL_CHECKOUT && python tooltool/tooltool.py --url $TOOLTOOL_SERVER -m $SRCDIR/$TOOLTOOL_MANIFEST fetch ${TOOLTOOL_CACHE:+ -c $TOOLTOOL_CACHE})