From 8edb6105b8611e0ca1144a8e380e15b375e197b1 Mon Sep 17 00:00:00 2001 From: Jesse Schwartzentruber Date: Thu, 8 Feb 2018 17:16:41 -0500 Subject: [PATCH] Bug 1421728 - Add a macosx64 fuzzing-asan build. r=dustin,froydnj MozReview-Commit-ID: DNNu4jyG50Z --HG-- extra : rebase_source : 4440c958965ee6021a3aaf732f9a87cc10763245 --- browser/config/mozconfigs/linux32/debug-asan | 1 + .../config/mozconfigs/linux32/nightly-asan | 1 + browser/config/mozconfigs/linux64/debug-asan | 1 + .../config/mozconfigs/linux64/nightly-asan | 1 + .../mozconfigs/linux64/nightly-asan-reporter | 1 + .../mozconfigs/linux64/nightly-fuzzing-asan | 3 +- .../config/mozconfigs/macosx64/nightly-asan | 5 +++ .../mozconfigs/macosx64/nightly-fuzzing-asan | 2 ++ build/unix/mozconfig.asan | 1 - build/unix/mozconfig.fuzzing | 28 ---------------- build/unix/rewrite_asan_dylib.py | 32 +++++++++++------- taskcluster/ci/build/macosx.yml | 33 +++++++++++++++++++ .../64_cross_fuzzing_asan.py | 31 +++++++++++++++++ .../mozharness/mozilla/building/buildbase.py | 1 + 14 files changed, 99 insertions(+), 42 deletions(-) create mode 100644 browser/config/mozconfigs/macosx64/nightly-fuzzing-asan delete mode 100644 build/unix/mozconfig.fuzzing create mode 100644 testing/mozharness/configs/builds/releng_sub_mac_configs/64_cross_fuzzing_asan.py diff --git a/browser/config/mozconfigs/linux32/debug-asan b/browser/config/mozconfigs/linux32/debug-asan index 72acc89c1d86..ddf79cb5f6ac 100644 --- a/browser/config/mozconfigs/linux32/debug-asan +++ b/browser/config/mozconfigs/linux32/debug-asan @@ -9,6 +9,7 @@ ac_add_options --enable-optimize="-O1" ac_add_options --enable-valgrind . $topsrcdir/build/unix/mozconfig.asan +ac_add_options --disable-elf-hack # Enable Telemetry export MOZ_TELEMETRY_REPORTING=1 diff --git a/browser/config/mozconfigs/linux32/nightly-asan b/browser/config/mozconfigs/linux32/nightly-asan index 629457c34cfc..7a3868dfc815 100644 --- a/browser/config/mozconfigs/linux32/nightly-asan +++ b/browser/config/mozconfigs/linux32/nightly-asan @@ -8,6 +8,7 @@ ac_add_options --enable-optimize="-O2 -g" ac_add_options --enable-valgrind . $topsrcdir/build/unix/mozconfig.asan +ac_add_options --disable-elf-hack # Package js shell. export MOZ_PACKAGE_JSSHELL=1 diff --git a/browser/config/mozconfigs/linux64/debug-asan b/browser/config/mozconfigs/linux64/debug-asan index 72acc89c1d86..ddf79cb5f6ac 100644 --- a/browser/config/mozconfigs/linux64/debug-asan +++ b/browser/config/mozconfigs/linux64/debug-asan @@ -9,6 +9,7 @@ ac_add_options --enable-optimize="-O1" ac_add_options --enable-valgrind . $topsrcdir/build/unix/mozconfig.asan +ac_add_options --disable-elf-hack # Enable Telemetry export MOZ_TELEMETRY_REPORTING=1 diff --git a/browser/config/mozconfigs/linux64/nightly-asan b/browser/config/mozconfigs/linux64/nightly-asan index 15bf38d6c054..b0a1ae5fe808 100644 --- a/browser/config/mozconfigs/linux64/nightly-asan +++ b/browser/config/mozconfigs/linux64/nightly-asan @@ -8,6 +8,7 @@ ac_add_options --enable-optimize="-O2 -gline-tables-only" ac_add_options --enable-valgrind . $topsrcdir/build/unix/mozconfig.asan +ac_add_options --disable-elf-hack # Package js shell. export MOZ_PACKAGE_JSSHELL=1 diff --git a/browser/config/mozconfigs/linux64/nightly-asan-reporter b/browser/config/mozconfigs/linux64/nightly-asan-reporter index 3512ae9b5519..8ba405cc7869 100644 --- a/browser/config/mozconfigs/linux64/nightly-asan-reporter +++ b/browser/config/mozconfigs/linux64/nightly-asan-reporter @@ -8,6 +8,7 @@ ac_add_options --enable-optimize="-O2 -gline-tables-only" ac_add_options --enable-valgrind . $topsrcdir/build/unix/mozconfig.asan +ac_add_options --disable-elf-hack ac_add_options --enable-address-sanitizer-reporter diff --git a/browser/config/mozconfigs/linux64/nightly-fuzzing-asan b/browser/config/mozconfigs/linux64/nightly-fuzzing-asan index 0ee0278251da..6850593611ac 100644 --- a/browser/config/mozconfigs/linux64/nightly-fuzzing-asan +++ b/browser/config/mozconfigs/linux64/nightly-fuzzing-asan @@ -12,7 +12,8 @@ ac_add_options "MOZ_ALLOW_LEGACY_EXTENSIONS=1" # ASan specific options on Linux ac_add_options --enable-valgrind -. $topsrcdir/build/unix/mozconfig.fuzzing +. $topsrcdir/build/unix/mozconfig.asan +ac_add_options --disable-elf-hack ac_add_options --enable-fuzzing unset MOZ_STDCXX_COMPAT diff --git a/browser/config/mozconfigs/macosx64/nightly-asan b/browser/config/mozconfigs/macosx64/nightly-asan index 919772ed5e6f..806b17f826a7 100644 --- a/browser/config/mozconfigs/macosx64/nightly-asan +++ b/browser/config/mozconfigs/macosx64/nightly-asan @@ -16,4 +16,9 @@ export MOZ_PACKAGE_JSSHELL=1 export MOZ_PKG_SPECIAL=asan . "$topsrcdir/build/macosx/mozconfig.common" + +# This is disabled by mozconfig.asan and reenabled by mozconfig.common. +# Ensure it is disabled since it conflicts with ASan. +ac_add_options --disable-crashreporter + . "$topsrcdir/build/mozconfig.common.override" diff --git a/browser/config/mozconfigs/macosx64/nightly-fuzzing-asan b/browser/config/mozconfigs/macosx64/nightly-fuzzing-asan new file mode 100644 index 000000000000..33701fb6ecf2 --- /dev/null +++ b/browser/config/mozconfigs/macosx64/nightly-fuzzing-asan @@ -0,0 +1,2 @@ +. "$topsrcdir/browser/config/mozconfigs/macosx64/nightly-asan" +ac_add_options --enable-fuzzing diff --git a/build/unix/mozconfig.asan b/build/unix/mozconfig.asan index 54cc96dc59a6..dbf6840e1ff2 100644 --- a/build/unix/mozconfig.asan +++ b/build/unix/mozconfig.asan @@ -22,7 +22,6 @@ ac_add_options --enable-debug-symbols ac_add_options --disable-install-strip ac_add_options --disable-jemalloc ac_add_options --disable-crashreporter -ac_add_options --disable-elf-hack ac_add_options --disable-profiling . "$topsrcdir/build/unix/mozconfig.stdcxx" diff --git a/build/unix/mozconfig.fuzzing b/build/unix/mozconfig.fuzzing deleted file mode 100644 index 54cc96dc59a6..000000000000 --- a/build/unix/mozconfig.fuzzing +++ /dev/null @@ -1,28 +0,0 @@ -MOZ_AUTOMATION_L10N_CHECK=0 - -. "$topsrcdir/build/mozconfig.common" - -# Use Clang as specified in manifest -export CC="$topsrcdir/clang/bin/clang -fgnu89-inline" -export CXX="$topsrcdir/clang/bin/clang++" -export LLVM_SYMBOLIZER="$topsrcdir/clang/bin/llvm-symbolizer" - -# Use a newer binutils, from the tooltool gcc package, if it's there -if [ -e "$topsrcdir/gcc/bin/ld" ]; then - export CC="$CC -B $topsrcdir/gcc/bin" - export CXX="$CXX -B $topsrcdir/gcc/bin" -fi - -# Enable ASan specific code and build workarounds -ac_add_options --enable-address-sanitizer - -# Mandatory options required for ASan builds (both on Linux and Mac) -export MOZ_DEBUG_SYMBOLS=1 -ac_add_options --enable-debug-symbols -ac_add_options --disable-install-strip -ac_add_options --disable-jemalloc -ac_add_options --disable-crashreporter -ac_add_options --disable-elf-hack -ac_add_options --disable-profiling - -. "$topsrcdir/build/unix/mozconfig.stdcxx" diff --git a/build/unix/rewrite_asan_dylib.py b/build/unix/rewrite_asan_dylib.py index 5ced21db4652..5d866dafd7b7 100644 --- a/build/unix/rewrite_asan_dylib.py +++ b/build/unix/rewrite_asan_dylib.py @@ -45,7 +45,7 @@ def resolve_rpath(filename): return path sys.stderr.write('@rpath could not be resolved from %s\n' % filename) - exit(1) + sys.exit(1) def scan_directory(path): dylibCopied = False @@ -60,37 +60,45 @@ def scan_directory(path): try: otoolOut = subprocess.check_output([substs['OTOOL'], '-L', filename]) - except: + except Exception: # Errors are expected on non-mach executables, ignore them and continue continue for line in otoolOut.splitlines(): - if line.find(DYLIB_NAME) != -1: + if DYLIB_NAME in line: absDylibPath = line.split()[0] # Don't try to rewrite binaries twice - if absDylibPath.find('@executable_path/') == 0: + if absDylibPath.startswith('@executable_path/'): continue if not dylibCopied: - if absDylibPath.find('@rpath/') == 0: + if absDylibPath.startswith('@rpath/'): rpath = resolve_rpath(filename) copyDylibPath = absDylibPath.replace('@rpath', rpath) else: copyDylibPath = absDylibPath - # Copy the runtime once to the main directory, which is passed - # as the argument to this function. - shutil.copy(copyDylibPath, path) + if os.path.isfile(copyDylibPath): + # Copy the runtime once to the main directory, which is passed + # as the argument to this function. + shutil.copy(copyDylibPath, path) - # Now rewrite the library itself - subprocess.check_call([substs['INSTALL_NAME_TOOL'], '-id', '@executable_path/' + DYLIB_NAME, os.path.join(path, DYLIB_NAME)]) - dylibCopied = True + # Now rewrite the library itself + subprocess.check_call([substs['INSTALL_NAME_TOOL'], '-id', '@executable_path/' + DYLIB_NAME, os.path.join(path, DYLIB_NAME)]) + dylibCopied = True + else: + sys.stderr.write('dylib path in %s was not found at: %s\n' % (filename, copyDylibPath)) # Now use install_name_tool to rewrite the path in our binary - subprocess.check_call([substs['INSTALL_NAME_TOOL'], '-change', absDylibPath, '@executable_path/' + DYLIB_NAME, filename]) + relpath = '' if path == root else os.path.relpath(path, root) + '/' + subprocess.check_call([substs['INSTALL_NAME_TOOL'], '-change', absDylibPath, '@executable_path/' + relpath + DYLIB_NAME, filename]) break + if not dylibCopied: + sys.stderr.write('%s could not be found\n' % DYLIB_NAME) + sys.exit(1) + if __name__ == '__main__': for d in sys.argv[1:]: scan_directory(d) diff --git a/taskcluster/ci/build/macosx.yml b/taskcluster/ci/build/macosx.yml index c0cb2fdd1c2a..4a6541a3c3bd 100644 --- a/taskcluster/ci/build/macosx.yml +++ b/taskcluster/ci/build/macosx.yml @@ -63,6 +63,39 @@ macosx64/opt: - linux64-rust-macos - linux64-sccache +macosx64-asan-fuzzing/opt: + description: "MacOS X x64 Cross-compile Fuzzing ASAN" + index: + product: firefox + job-name: macosx64-fuzzing-asan-opt + treeherder: + platform: osx-cross/asan + symbol: Bof + worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64 + worker: + max-run-time: 36000 + env: + PERFHERDER_EXTRA_OPTIONS: asan-fuzzing + TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-releng.manifest" + run: + using: mozharness + actions: [get-secrets build update] + config: + - builds/releng_base_firefox.py + - builds/releng_base_mac_64_cross_builds.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: cross-fuzzing-asan + tooltool-downloads: internal + toolchains: + - linux64-cctools-port + - linux64-clang-6-pre-macosx-cross + - linux64-hfsplus + - linux64-libdmg + - linux64-llvm-dsymutil + - linux64-rust-macos + - linux64-sccache + macosx64-dmd/opt: description: "MacOS X x64 DMD Cross-compile" index: diff --git a/testing/mozharness/configs/builds/releng_sub_mac_configs/64_cross_fuzzing_asan.py b/testing/mozharness/configs/builds/releng_sub_mac_configs/64_cross_fuzzing_asan.py new file mode 100644 index 000000000000..8d16957b3d33 --- /dev/null +++ b/testing/mozharness/configs/builds/releng_sub_mac_configs/64_cross_fuzzing_asan.py @@ -0,0 +1,31 @@ +import os + +config = { + 'default_actions': [ + 'clobber', + 'build', + ], + 'stage_platform': 'macosx64-fuzzing-asan', + 'publish_nightly_en_US_routes': False, + 'build_type': 'asan', + 'platform_supports_post_upload_to_latest': False, + 'enable_signing': False, + #### 64 bit build specific ##### + 'env': { + 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'), + 'DISPLAY': ':2', + 'HG_SHARE_BASE_DIR': '/builds/hg-shared', + 'MOZ_OBJDIR': '%(abs_obj_dir)s', + 'TINDERBOX_OUTPUT': '1', + 'TOOLTOOL_CACHE': '/builds/tooltool_cache', + 'TOOLTOOL_HOME': '/builds', + 'MOZ_CRASHREPORTER_NO_REPORT': '1', + 'LC_ALL': 'C', + 'ASAN_OPTIONS': 'detect_leaks=0', + ## 64 bit specific + 'PATH': '/usr/local/bin:/usr/lib64/ccache:/bin:\ +/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/tools/git/bin:/tools/python27/bin:\ +/tools/python27-mercurial/bin:/home/cltbld/bin', + }, + 'mozconfig_variant': 'nightly-fuzzing-asan', +} diff --git a/testing/mozharness/mozharness/mozilla/building/buildbase.py b/testing/mozharness/mozharness/mozilla/building/buildbase.py index ffe2ca62f28d..eedaaa5629e0 100755 --- a/testing/mozharness/mozharness/mozilla/building/buildbase.py +++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py @@ -414,6 +414,7 @@ class BuildOptionParser(object): 'cross-debug-searchfox': 'builds/releng_sub_%s_configs/%s_cross_debug_searchfox.py', 'cross-debug-artifact': 'builds/releng_sub_%s_configs/%s_cross_debug_artifact.py', 'cross-noopt-debug': 'builds/releng_sub_%s_configs/%s_cross_noopt_debug.py', + 'cross-fuzzing-asan': 'builds/releng_sub_%s_configs/%s_cross_fuzzing_asan.py', 'cross-artifact': 'builds/releng_sub_%s_configs/%s_cross_artifact.py', 'debug': 'builds/releng_sub_%s_configs/%s_debug.py', 'fuzzing-debug': 'builds/releng_sub_%s_configs/%s_fuzzing_debug.py',