Bug 1475562 Produce pdbs for the mingw-clang build job r=ted

This patch also changes how pdbs for the ASAN job are copied:
we relax restrictions so that pdbs if present) are always copied out
and add an environment variable MOZ_COPY_PDBS to indicate when we
want to produce pdbs for copying.

--HG--
extra : rebase_source : ae91be712f9fdc2e897f4cf92b601e39494e6e57

browser/config/mozconfigs/win32/mingwclang

@@ -30,6 +30,7 @@ ac_add_options --target=i686-w64-mingw32
 ac_add_options --with-toolchain-prefix=i686-w64-mingw32-
 
 ac_add_options --disable-warnings-as-errors
+MOZ_COPY_PDBS=1
 
 # Temporary config settings until we get these working on mingw
 ac_add_options --disable-accessibility # https://sourceforge.net/p/mingw-w64/bugs/648/

browser/config/mozconfigs/win64/mingwclang

@@ -30,6 +30,7 @@ ac_add_options --target=x86_64-w64-mingw32
 ac_add_options --with-toolchain-prefix=x86_64-w64-mingw32-
 
 ac_add_options --disable-warnings-as-errors
+MOZ_COPY_PDBS=1
 
 # Temporary config settings until we get these working on mingw
 ac_add_options --disable-accessibility # https://sourceforge.net/p/mingw-w64/bugs/648/

build/moz.configure/toolchain.configure

@@ -1142,11 +1142,13 @@ def bindgen_cflags(value):
 add_old_configure_assignment('_BINDGEN_CFLAGS', bindgen_cflags)
 
 
-@depends(c_compiler)
-def default_debug_flags(compiler_info):
+@depends(c_compiler, target)
+def default_debug_flags(compiler_info, target):
     # Debug info is ON by default.
     if compiler_info.type in ('msvc', 'clang-cl'):
         return '-Zi'
+    elif target.kernel == 'WINNT' and compiler_info.type == 'clang':
+        return '-g -gcodeview'
     return '-g'
 
 
@@ -1993,11 +1995,16 @@ add_old_configure_assignment('LIBFUZZER_FLAGS', libfuzzer_flags.use_flags)
 @depends(target, c_compiler)
 def make_shared_library(target, compiler):
     if target.os == 'WINNT':
-        if compiler.type in ('gcc', 'clang'):
+        if compiler.type == 'gcc':
             return namespace(
                 mkshlib=['$(CXX)', '$(DSO_LDOPTS)', '-o', '$@'],
                 mkcshlib=['$(CC)', '$(DSO_LDOPTS)', '-o', '$@'],
             )
+        elif compiler.type == 'clang':
+            return namespace(
+                mkshlib=['$(CXX)', '$(DSO_LDOPTS)', '-Wl,-pdb,$(LINK_PDBFILE)', '-o', '$@'],
+                mkcshlib=['$(CC)', '$(DSO_LDOPTS)', '-Wl,-pdb,$(LINK_PDBFILE)', '-o', '$@'],
+            )
         else:
             linker = [
                 '$(LINKER)',

build/win64/mozconfig.asan

@@ -7,6 +7,7 @@ if [ -d "$topsrcdir/clang" ]; then
     mk_export_correct_style LIB
     export LDFLAGS="clang_rt.asan_dynamic-x86_64.lib clang_rt.asan_dynamic_runtime_thunk-x86_64.lib"
 
+    export MOZ_COPY_PDBS=1
     export LLVM_SYMBOLIZER="$topsrcdir/clang/bin/llvm-symbolizer.exe"
     export MOZ_CLANG_RT_ASAN_LIB_PATH="${CLANG_LIB_DIR}/clang_rt.asan_dynamic-x86_64.dll"
 fi

config/rules.mk

@@ -126,6 +126,9 @@ endif # MKSHLIB
 endif # FORCE_SHARED_LIB
 
 ifeq ($(OS_ARCH),WINNT)
+
+LINK_PDBFILE ?= $(basename $(@F)).pdb
+
 ifndef GNU_CC
 
 #
@@ -147,7 +150,6 @@ endif
 COMPILE_CFLAGS += $(COMPILE_PDB_FLAG)
 COMPILE_CXXFLAGS += $(COMPILE_PDB_FLAG)
 
-LINK_PDBFILE ?= $(basename $(@F)).pdb
 ifdef MOZ_DEBUG
 CODFILE=$(basename $(@F)).cod
 endif
@@ -161,6 +163,12 @@ MOZ_PROGRAM_LDFLAGS += -Wl,-rpath -Wl,@executable_path/Frameworks
 endif
 endif
 
+ifeq ($(OS_ARCH),WINNT)
+ifeq ($(CC_TYPE),clang)
+MOZ_PROGRAM_LDFLAGS += -Wl,-pdb,$(dir $@)/$(LINK_PDBFILE)
+endif
+endif
+
 ifeq ($(HOST_OS_ARCH),WINNT)
 HOST_PDBFILE=$(basename $(@F)).pdb
 HOST_PDB_FLAG ?= -Fd$(HOST_PDBFILE)
@@ -819,13 +827,13 @@ DUMP_SYMS_TARGETS :=
 endif
 endif
 
-ifdef MOZ_CRASHREPORTER
-$(foreach file,$(DUMP_SYMS_TARGETS),$(eval $(call syms_template,$(file),$(notdir $(file))_syms.track)))
-else ifneq (,$(and $(LLVM_SYMBOLIZER),$(filter WINNT,$(OS_ARCH)),$(MOZ_AUTOMATION)))
+ifdef MOZ_COPY_PDBS
 PDB_FILES = $(addsuffix .pdb,$(basename $(DUMP_SYMS_TARGETS)))
 PDB_DEST ?= $(FINAL_TARGET)
 PDB_TARGET = syms
 INSTALL_TARGETS += PDB
+else ifdef MOZ_CRASHREPORTER
+$(foreach file,$(DUMP_SYMS_TARGETS),$(eval $(call syms_template,$(file),$(notdir $(file))_syms.track)))
 endif
 
 cargo_host_flag := --target=$(RUST_HOST_TARGET)

moz.configure

@@ -39,6 +39,12 @@ def imply_disable_compile_environment(value):
     if value:
         return False
 
+option(env='MOZ_COPY_PDBS',
+    help='For builds that do not support symbols in the normal fashion,'
+         ' generate and copy them into the resulting build archive.')
+
+set_config('MOZ_COPY_PDBS', depends_if('MOZ_COPY_PDBS')(lambda _: True))
+
 imply_option('--enable-compile-environment', imply_disable_compile_environment)
 
 option('--disable-compile-environment',

python/mozbuild/mozbuild/action/check_binary.py

@@ -39,6 +39,7 @@ TARGET = {
     'readelf': '{}readelf'.format(
         buildconfig.substs.get('TOOLCHAIN_PREFIX', '')),
     'nm': '{}nm'.format(buildconfig.substs.get('TOOLCHAIN_PREFIX', '')),
+    'readobj': '{}readobj'.format(buildconfig.substs.get('TOOLCHAIN_PREFIX', '')),
 }
 
 if buildconfig.substs.get('HAVE_64BIT_BUILD'):
@@ -190,7 +191,24 @@ def check_nsmodules(target, binary):
                     symbols.append((int(data[2], 16), GUESSED_NSMODULE_SIZE,
                                     name))
     else:
-        for line in get_output(target['nm'], '-P', binary):
+        # MinGW-Clang, when building pdbs, doesn't include the symbol table into
+        # the final module. To get the NSModule info, we can look at the exported
+        # symbols. (#1475562)
+        if buildconfig.substs['OS_ARCH'] == 'WINNT' and \
+           buildconfig.substs['HOST_OS_ARCH'] != 'WINNT':
+            readobj_output = get_output(target['readobj'], '-coff-exports', binary)
+            # Transform the output of readobj into nm-like output
+            output = []
+            for line in readobj_output:
+                if "Name" in line:
+                    name = line.replace("Name:", "").strip()
+                elif "RVA" in line:
+                    rva = line.replace("RVA:", "").strip()
+                    output.append("%s r %s" % (name, rva))
+        else:
+            output = get_output(target['nm'], '-P', binary)
+
+        for line in output:
             data = line.split()
             # Some symbols may not have a size listed at all.
             if len(data) == 3:

taskcluster/scripts/misc/build-clang-7-mingw.sh

@@ -275,6 +275,11 @@ build_windres() {
   # Manually install only nm and windres
   cp binutils/windres $INSTALL_DIR/bin/$machine-w64-mingw32-windres
   cp binutils/nm-new $INSTALL_DIR/bin/$machine-w64-mingw32-nm
+
+  pushd $INSTALL_DIR/bin/
+  ln -s llvm-readobj $machine-w64-mingw32-readobj
+  popd
+
   popd
 }
 

toolkit/mozapps/installer/packager.py

@@ -311,8 +311,9 @@ def main():
                                LibSignFile(os.path.join(args.destination,
                                                         libname)))
 
-    # Include pdb files for llvm-symbolizer to resolve symbols.
-    if buildconfig.substs.get('LLVM_SYMBOLIZER') and mozinfo.isWin:
+    # If a pdb file is present and we were instructed to copy it, include it.
+    # Run on all OSes to capture MinGW builds
+    if buildconfig.substs['MOZ_COPY_PDBS']:
         for p, f in copier:
             if isinstance(f, ExecutableFile):
                 pdbname = os.path.splitext(f.inputs()[0])[0] + '.pdb'