Bug 1740187 - Annotate crash reports with rlbox sandbox malloc failures r=bholley

Differential Revision: https://phabricator.services.mozilla.com/D130727

config/external/rlbox_wasm2c_sandbox/rlbox_wasm2c_thread_locals.cpp

@@ -15,6 +15,8 @@
 
 #  include "mozilla/rlbox/rlbox.hpp"
 
+#  include "nsExceptionHandler.h"
+
 // The MingW compiler does not correctly handle static thread_local inline
 // members. We instead TLS storage via functions. This can be removed if the
 // MingW bug is fixed.
@@ -27,6 +29,16 @@ extern "C" {
 void moz_wasm2c_trap_handler(const char* msg) {
   MOZ_CRASH_UNSAFE_PRINTF("wasm2c crash: %s", msg);
 }
+
+// The below function is called if a malloc in sandboxed code returns null
+// This indicates that the sandbox has run out of memory.
+void moz_wasm2c_malloc_failed(uint32_t size) {
+  // We don't use the allocation size information for now
+  (void) size;
+
+  CrashReporter::AnnotateCrashReport(
+      CrashReporter::Annotation::WasmLibrarySandboxMallocFailed, true);
+}
 }
 
 #endif

config/external/wasm2c_sandbox_compiler/moz.yaml

@@ -9,8 +9,8 @@ origin:
   description: wasm2c fork used for rlbox sandboxing
   url: https://github.com/PLSysSec/wasm2c_sandbox_compiler
 
-  release: commit 504848a4b1deb8f3d5664edb21ad1dc073863d6c (2021-11-06T05:17:34Z).
-  revision: 504848a4b1deb8f3d5664edb21ad1dc073863d6c
+  release: commit cdcf20186f3bfef472b32836b10e12b5cdaaebda (2021-11-09T07:39:20Z).
+  revision: cdcf20186f3bfef472b32836b10e12b5cdaaebda
 
   license: Apache-2.0
   license-file: LICENSE

security/rlbox/moz.build

@@ -23,6 +23,10 @@ SOURCES += [
 # Configure the wasm runtime to use a custom trap handler that calls MOZ_CRASH
 DEFINES["WASM_RT_CUSTOM_TRAP_HANDLER"] = "moz_wasm2c_trap_handler"
 
+# Configure the wasm runtime to invoke a callback when a malloc fails inside
+# the sandbox. This information is used to annotate crash reports
+DEFINES["WASM2C_MALLOC_FAIL_CALLBACK"] = "moz_wasm2c_malloc_failed"
+
 # Configuration that removes the wasm2c functions from shared library exports
 DEFINES["WASM_DONT_EXPORT_FUNCS"] = True
 

third_party/wasm2c/src/c-writer.cc

@@ -1440,6 +1440,7 @@ void CWriter::Write(const Func& func) {
     Write(GetFuncStaticOrExport(out_func_name), "u32 w2c_dlmalloc(wasm2c_sandbox_t* const sbx, u32 ptr_size) ", OpenBrace());
     Write("u32 ret = w2c_dlmalloc_wrapped(sbx, ptr_size);", Newline());
     Write("WASM2C_SHADOW_MEMORY_DLMALLOC(&(sbx->", memory_name, "), ret, ptr_size);", Newline());
+    Write("WASM2C_MALLOC_FAIL_CHECK(ret, ptr_size);", Newline());
     Write("return ret;", Newline());
     Write(CloseBrace());
   } else if (out_func_name == "w2c_dlfree") {

third_party/wasm2c/src/prebuilt/wasm2c.include.c

@@ -63,6 +63,16 @@ const char SECTION_NAME(declarations)[] =
 "    wasm_rt_callback_error_trap(&table, x, func_types[ft]);                                          \\\n"
 "  }\n"
 "\n"
+"#if defined(WASM2C_MALLOC_FAIL_CALLBACK)\n"
+"void WASM2C_MALLOC_FAIL_CALLBACK(u32 ptr_size);\n"
+"# define WASM2C_MALLOC_FAIL_CHECK(ptr, ptr_size)  \\\n"
+"  if (!ptr) {                                     \\\n"
+"    WASM2C_MALLOC_FAIL_CALLBACK(ptr_size);        \\\n"
+"  }\n"
+"#else\n"
+"# define WASM2C_MALLOC_FAIL_CHECK(ptr, ptr_size)\n"
+"#endif\n"
+"\n"
 "#if defined(WASM_CHECK_SHADOW_MEMORY)\n"
 "#  define WASM2C_SHADOW_MEMORY_LOAD(mem, func_name, ptr, ptr_size)  wasm2c_shadow_memory_load(mem, func_name, ptr, ptr_size)\n"
 "#  define WASM2C_SHADOW_MEMORY_STORE(mem, func_name, ptr, ptr_size) wasm2c_shadow_memory_store(mem, func_name, ptr, ptr_size)\n"

third_party/wasm2c/src/wasm2c.c.tmpl

@@ -60,6 +60,16 @@
     wasm_rt_callback_error_trap(&table, x, func_types[ft]);                                          \
   }
 
+#if defined(WASM2C_MALLOC_FAIL_CALLBACK)
+void WASM2C_MALLOC_FAIL_CALLBACK(u32 ptr_size);
+# define WASM2C_MALLOC_FAIL_CHECK(ptr, ptr_size)  \
+  if (!ptr) {                                     \
+    WASM2C_MALLOC_FAIL_CALLBACK(ptr_size);        \
+  }
+#else
+# define WASM2C_MALLOC_FAIL_CHECK(ptr, ptr_size)
+#endif
+
 #if defined(WASM_CHECK_SHADOW_MEMORY)
 #  define WASM2C_SHADOW_MEMORY_LOAD(mem, func_name, ptr, ptr_size)  wasm2c_shadow_memory_load(mem, func_name, ptr, ptr_size)
 #  define WASM2C_SHADOW_MEMORY_STORE(mem, func_name, ptr, ptr_size) wasm2c_shadow_memory_store(mem, func_name, ptr, ptr_size)

toolkit/crashreporter/CrashAnnotations.yaml

@@ -952,6 +952,13 @@ VRProcessStatus:
     Status of the VR process, can be set to "Running" or "Destroyed"
   type: string
 
+WasmLibrarySandboxMallocFailed:
+  description: >
+    Set to 1 if a rlbox wasm library sandbox ran out of memory, causing a
+    malloc inside the sandbox to fail.
+  type: boolean
+  ping: true
+
 WindowsErrorReporting:
   description: >
     Set to 1 if this crash was intercepted via the Windows Error Reporting