Bug 1547519 - Fix jemalloc redirections for MinGW build r=glandium

In the MinGW build, calls to malloc inside mozglue were not being
redirected as defined in the .def file. We create aliases for the
redirected functions to correctly redirect them inside mozglue.

An alternate solution for this exists. Rather than creating the
importlib during the linking step for mozglue, we could have used
dlltool to create it, and then provided it during linking. This
would allow mozglue to know that it should redirect calls to malloc
to je_malloc as specified in the .def file.

Differential Revision: https://phabricator.services.mozilla.com/D38407

--HG--
extra : moz-landing-system : lando

browser/config/mozconfigs/win32/mingwclang

@@ -34,9 +34,6 @@ ac_add_options --with-toolchain-prefix=i686-w64-mingw32-
 ac_add_options --disable-warnings-as-errors
 MOZ_COPY_PDBS=1
 
-# Bug 1547519
-ac_add_options --disable-jemalloc
-
 # This replicates Tor's configuration
 ac_add_options --enable-proxy-bypass-protection
 

browser/config/mozconfigs/win64/mingwclang

@@ -34,9 +34,6 @@ ac_add_options --with-toolchain-prefix=x86_64-w64-mingw32-
 ac_add_options --disable-warnings-as-errors
 MOZ_COPY_PDBS=1
 
-# Bug 1547519
-ac_add_options --disable-jemalloc
-
 # This replicates Tor's configuration
 ac_add_options --enable-proxy-bypass-protection
 

memory/build/mozjemalloc.cpp

@@ -130,6 +130,7 @@
 #include "mozilla/Attributes.h"
 #include "mozilla/CheckedInt.h"
 #include "mozilla/DoublyLinkedList.h"
+#include "mozilla/HelperMacros.h"
 #include "mozilla/Likely.h"
 #include "mozilla/MathAlgorithms.h"
 #include "mozilla/RandomNum.h"
@@ -4672,14 +4673,29 @@ static void replace_malloc_init_funcs(malloc_table_t* table) {
 #endif  // MOZ_REPLACE_MALLOC
 // ***************************************************************************
 // Definition of all the _impl functions
+// GENERIC_MALLOC_DECL2_MINGW is only used for the MinGW build, and aliases
+// the malloc funcs (e.g. malloc) to the je_ versions. It does not generate
+// aliases for the other functions (jemalloc and arena functions).
+//
+// We do need aliases for the other mozglue.def-redirected functions though,
+// these are done at the bottom of mozmemory_wrap.cpp
+#define GENERIC_MALLOC_DECL2_MINGW(name, name_impl, return_type, ...) \
+  return_type name(ARGS_HELPER(TYPED_ARGS, ##__VA_ARGS__))            \
+      __attribute__((alias(MOZ_STRINGIFY(name_impl))));
 
 #define GENERIC_MALLOC_DECL2(name, name_impl, return_type, ...)   \
   return_type name_impl(ARGS_HELPER(TYPED_ARGS, ##__VA_ARGS__)) { \
     return DefaultMalloc::name(ARGS_HELPER(ARGS, ##__VA_ARGS__)); \
   }
 
-#define GENERIC_MALLOC_DECL(name, return_type, ...) \
-  GENERIC_MALLOC_DECL2(name, name##_impl, return_type, ##__VA_ARGS__)
+#ifndef __MINGW32__
+#  define GENERIC_MALLOC_DECL(name, return_type, ...) \
+    GENERIC_MALLOC_DECL2(name, name##_impl, return_type, ##__VA_ARGS__)
+#else
+#  define GENERIC_MALLOC_DECL(name, return_type, ...)                   \
+    GENERIC_MALLOC_DECL2(name, name##_impl, return_type, ##__VA_ARGS__) \
+    GENERIC_MALLOC_DECL2_MINGW(name, name##_impl, return_type, ##__VA_ARGS__)
+#endif
 
 #define MALLOC_DECL(...) \
   MOZ_MEMORY_API MACRO_CALL(GENERIC_MALLOC_DECL, (__VA_ARGS__))

memory/build/mozmemory_wrap.cpp

@@ -99,4 +99,40 @@ MOZ_MEMORY_API wchar_t* wcsdup_impl(const wchar_t* src) {
 MOZ_MEMORY_API void* _aligned_malloc_impl(size_t size, size_t alignment) {
   return memalign_impl(alignment, size);
 }
+
+#  ifdef __MINGW32__
+MOZ_BEGIN_EXTERN_C
+// As in mozjemalloc.cpp, we generate aliases for functions
+// redirected in mozglue.def
+void* _aligned_malloc(size_t size, size_t alignment)
+    __attribute__((alias(MOZ_STRINGIFY(_aligned_malloc_impl))));
+void _aligned_free(void* aPtr) __attribute__((alias(MOZ_STRINGIFY(free_impl))));
+
+char* strndup(const char* src, size_t len)
+    __attribute__((alias(MOZ_STRINGIFY(strdup_impl))));
+char* strdup(const char* src)
+    __attribute__((alias(MOZ_STRINGIFY(strdup_impl))));
+char* _strdup(const char* src)
+    __attribute__((alias(MOZ_STRINGIFY(strdup_impl))));
+wchar_t* wcsdup(const wchar_t* src)
+    __attribute__((alias(MOZ_STRINGIFY(wcsdup_impl))));
+wchar_t* _wcsdup(const wchar_t* src)
+    __attribute__((alias(MOZ_STRINGIFY(wcsdup_impl))));
+
+// libc++.a contains references to __imp__aligned_malloc (because it
+// is declared dllimport in the headers.)
+//
+// The linker sees jemalloc's _aligned_malloc symbol in your objects,
+// then libc++.a comes along and needs __imp__aligned_malloc, which
+// pulls in those parts of libucrt.a (or libmsvcrt.a in practice),
+// which define both __imp__aligned_malloc and _aligned_malloc, and
+// this causes a conflict.
+//
+// The fix is to define not only an _aligned_malloc symbol (via an
+// alias), but also define the __imp__aligned_malloc pointer to it.
+// This is done with __MINGW_IMP_SYMBOL to handle x86/x64 differences.
+void (*__MINGW_IMP_SYMBOL(_aligned_free))(void*) = _aligned_free;
+void* (*__MINGW_IMP_SYMBOL(_aligned_malloc))(size_t, size_t) = _aligned_malloc;
+MOZ_END_EXTERN_C
+#  endif
 #endif  // XP_WIN

memory/mozalloc/moz.build

@@ -37,9 +37,13 @@ SOURCES += [
 ]
 
 if CONFIG['MOZ_MEMORY']:
-    SOURCES += [
-        'cxxalloc.cpp',
-    ]
+    # In MinGW, we don't want to actually export these functions out of the library
+    # as the functions in libc++ correctly forward to jemalloc and exporting them
+    # produces duplicate symbol errors.
+    if not (CONFIG['CC_TYPE'] == 'clang' and CONFIG['OS_TARGET'] == 'WINNT'):
+        SOURCES += [
+            'cxxalloc.cpp',
+        ]
 
 if CONFIG['CPU_ARCH'] == 'arm' and CONFIG['CC_TYPE'] == 'clang':
     SOURCES['mozalloc_abort.cpp'].flags += ['-Wno-infinite-recursion']