Bug 1678152 - Catch all stack overflows on Linux r=jld

This patch adds a library that contains an interposer function for
pthread_create(). The interposer will setup an alternate signal stack to
handle crashes - thus enabling us to catch stack overflows - and then call the
real pthread_create() function. Since the interposer needs to appear in the
linker's search order before libpthread we manually link it into firefox,
plugin-container and xpcshell's executables ASAP.

Differential Revision: https://phabricator.services.mozilla.com/D132736

browser/app/moz.build

@@ -53,6 +53,11 @@ LOCAL_INCLUDES += [
     "/xpcom/build",
 ]
 
+# The pthred_create() interposer needs to be linked as early as possible so
+# that it will appear before libpthread when resolving symbols.
+if CONFIG["OS_ARCH"] == "Linux" and CONFIG["MOZ_CRASHREPORTER"]:
+    USE_LIBS += ["pthread_create_interposer"]
+
 if CONFIG["LIBFUZZER"]:
     USE_LIBS += ["fuzzer"]
     LOCAL_INCLUDES += [

ipc/app/moz.build

@@ -16,6 +16,11 @@ else:
         "MozillaRuntimeMain.cpp",
     ]
 
+# The pthred_create() interposer needs to be linked as early as possible so
+# that it will appear before libpthread when resolving symbols.
+if CONFIG["OS_ARCH"] == "Linux" and CONFIG["MOZ_CRASHREPORTER"]:
+    USE_LIBS += ["pthread_create_interposer"]
+
 include("/ipc/chromium/chromium-config.mozbuild")
 
 LOCAL_INCLUDES += [

js/xpconnect/shell/moz.build

@@ -10,6 +10,11 @@ SOURCES += [
     "xpcshell.cpp",
 ]
 
+# The pthred_create() interposer needs to be linked as early as possible so
+# that it will appear before libpthread when resolving symbols.
+if CONFIG["OS_ARCH"] == "Linux" and CONFIG["MOZ_CRASHREPORTER"]:
+    USE_LIBS += ["pthread_create_interposer"]
+
 if CONFIG["LIBFUZZER"]:
     USE_LIBS += ["fuzzer"]
 

toolkit/crashreporter/moz.build

@@ -53,6 +53,7 @@ if CONFIG["MOZ_CRASHREPORTER"]:
             "google-breakpad/src/common",
             "google-breakpad/src/common/linux",
             "google-breakpad/src/processor",
+            "pthread_create_interposer",
         ]
 
         if CONFIG["MOZ_OXIDIZED_BREAKPAD"]:

toolkit/crashreporter/pthread_create_interposer/moz.build

@@ -0,0 +1,12 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+Library("pthread_create_interposer")
+
+NoVisibilityFlags()
+
+UNIFIED_SOURCES += [
+    "pthread_create_interposer.cpp",
+]

toolkit/crashreporter/pthread_create_interposer/pthread_create_interposer.cpp

@@ -0,0 +1,119 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include <algorithm>
+
+#include <dlfcn.h>
+#include <pthread.h>
+#include <signal.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+
+#include "mozilla/Assertions.h"
+#include "mozilla/DebugOnly.h"
+
+using mozilla::DebugOnly;
+
+struct PthreadCreateParams {
+  void* (*start_routine)(void*);
+  void* arg;
+};
+
+const size_t kSigStackSize = std::max(size_t(16384), size_t(SIGSTKSZ));
+
+// Install the alternate signal stack, returns a pointer to the memory area we
+// mapped to store the stack only if it was installed successfully, otherwise
+// returns NULL.
+static void* install_sig_alt_stack() {
+  void* alt_stack_mem = mmap(nullptr, kSigStackSize, PROT_READ | PROT_WRITE,
+                             MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+  if (alt_stack_mem) {
+    stack_t alt_stack = {
+        .ss_sp = alt_stack_mem,
+        .ss_flags = 0,
+        .ss_size = kSigStackSize,
+    };
+
+    int rv = sigaltstack(&alt_stack, nullptr);
+    if (rv == 0) {
+      return alt_stack_mem;
+    }
+
+    rv = munmap(alt_stack_mem, kSigStackSize);
+    MOZ_ASSERT(rv == 0);
+  }
+
+  return nullptr;
+}
+
+// Uninstall the alternate signal handler and unmaps it. Does nothing if
+// alt_stack_mem is NULL.
+static void uninstall_sig_alt_stack(void* alt_stack_mem) {
+  if (alt_stack_mem) {
+    stack_t disable_alt_stack = {};
+    disable_alt_stack.ss_flags = SS_DISABLE;
+    DebugOnly<int> rv = sigaltstack(&disable_alt_stack, nullptr);
+    MOZ_ASSERT(rv == 0);
+    rv = munmap(alt_stack_mem, kSigStackSize);
+    MOZ_ASSERT(rv == 0);
+  }
+}
+
+// This replaces the routine passed to pthread_create() when a thread is
+// started, it handles the alternate signal stack and calls the thread's
+// actual routine.
+void* set_alt_signal_stack_and_start(PthreadCreateParams* params) {
+  void* (*start_routine)(void*) = params->start_routine;
+  void* arg = params->arg;
+  free(params);
+
+  void* thread_rv = nullptr;
+  void* alt_stack_mem = install_sig_alt_stack();
+  pthread_cleanup_push(uninstall_sig_alt_stack, alt_stack_mem);
+  thread_rv = start_routine(arg);
+  pthread_cleanup_pop(1);
+
+  return thread_rv;
+}
+
+using pthread_create_func_t = int (*)(pthread_t*, const pthread_attr_t*,
+                                      void* (*)(void*), void*);
+
+extern "C" {
+// This interposer replaces libpthread's pthread_create() so that we can
+// inject an alternate signal stack in every new thread.
+__attribute__((visibility("default"))) int pthread_create(
+    pthread_t* thread, const pthread_attr_t* attr,
+    void* (*start_routine)(void*), void* arg) {
+  // static const pthread_create_func_t real_pthread_create =
+  static const pthread_create_func_t real_pthread_create =
+      (pthread_create_func_t)dlsym(RTLD_NEXT, "pthread_create");
+
+  if (real_pthread_create == nullptr) {
+    MOZ_CRASH(
+        "pthread_create() interposition failed but the interposer function is "
+        "still being called, this won't work!");
+  }
+
+  if (real_pthread_create == pthread_create) {
+    MOZ_CRASH(
+        "We could not obtain the real pthread_create(). Calling the symbol we "
+        "got would make us enter an infinte loop so stop here instead.");
+  }
+
+  PthreadCreateParams* params =
+      (PthreadCreateParams*)malloc(sizeof(PthreadCreateParams));
+  params->start_routine = start_routine;
+  params->arg = arg;
+
+  int result = real_pthread_create(
+      thread, attr, (void* (*)(void*))set_alt_signal_stack_and_start, params);
+
+  if (result != 0) {
+    free(params);
+  }
+
+  return result;
+}
+}

toolkit/crashreporter/test/CrashTestUtils.jsm

@@ -36,6 +36,7 @@ var CrashTestUtils = {
   CRASH_PHC_BOUNDS_VIOLATION: 23,
   CRASH_HEAP_CORRUPTION: 24,
   CRASH_EXC_GUARD: 25,
+  CRASH_STACK_OVERFLOW: 26,
 
   // Constants for dumpHasStream()
   // From google_breakpad/common/minidump_format.h

toolkit/crashreporter/test/nsTestCrasher.cpp

@@ -97,6 +97,7 @@ const int16_t CRASH_PHC_DOUBLE_FREE = 22;
 const int16_t CRASH_PHC_BOUNDS_VIOLATION = 23;
 const int16_t CRASH_HEAP_CORRUPTION = 24;
 const int16_t CRASH_EXC_GUARD = 25;
+const int16_t CRASH_STACK_OVERFLOW = 26;
 
 #if XP_WIN && HAVE_64BIT_BUILD && defined(_M_X64) && !defined(__MINGW32__)
 
@@ -153,6 +154,23 @@ uint8_t* GetPHCAllocation(size_t aSize) {
 }
 #endif
 
+#ifndef XP_WIN
+static void* overflow_stack(void* aUnused) {
+  // We use a dummy variable and a bit of magic to pretend we care about what's
+  // on the stack so the compiler doesn't optimize the loop and allocations away
+  void* rv = nullptr;
+
+  for (size_t i = 0; i < 1024 * 1024 * 1024; i++) {
+    void* ptr = alloca(sizeof(void*));
+    if (ptr != nullptr) {
+      rv = ptr;
+    }
+  }
+
+  return rv;
+}
+#endif  // XP_WIN
+
 extern "C" NS_EXPORT void Crash(int16_t how) {
   switch (how) {
     case CRASH_INVALID_POINTER_DEREF: {
@@ -265,6 +283,17 @@ extern "C" NS_EXPORT void Crash(int16_t how) {
       }
     }
 #endif  // XP_MACOSX
+#ifndef XP_WIN
+    case CRASH_STACK_OVERFLOW: {
+      pthread_t thread_id;
+      int rv = pthread_create(&thread_id, nullptr, overflow_stack, nullptr);
+      if (!rv) {
+        pthread_join(thread_id, nullptr);
+      }
+
+      break;  // This should be unreachable
+    }
+#endif  // XP_WIN
     default:
       break;
   }

toolkit/crashreporter/test/unit/test_crash_stack_overflow.js

@@ -0,0 +1,21 @@
+add_task(async function run_test() {
+  if (!("@mozilla.org/toolkit/crash-reporter;1" in Cc)) {
+    dump(
+      "INFO | test_crash_stack_overflow.js | Can't test crashreporter in a non-libxul build.\n"
+    );
+    return;
+  }
+
+  // Try crashing by overflowing a thread's stack
+  await do_crash(
+    function() {
+      crashType = CrashTestUtils.CRASH_STACK_OVERFLOW;
+      crashReporter.annotateCrashReport("TestKey", "TestValue");
+    },
+    async function(mdump, extra, extraFile) {
+      Assert.equal(extra.TestKey, "TestValue");
+    },
+    // process will exit with a zero exit status
+    true
+  );
+});

toolkit/crashreporter/test/unit/xpcshell.ini

@@ -124,3 +124,6 @@ skip-if = !(os == 'win' && bits == 64 && processor == 'x86_64')
 reason = Windows test specific to the x86-64 architecture
 support-files = test_crash_win64cfi_not_a_pe.exe
 
+[test_crash_stack_overflow.js]
+skip-if = os != 'linux'
+reason = Still broken on macOS and not yet supported on Windows