Bug 640265 - Don't GC during OOM reporting (r=dmandelin)

js/src/jscntxt.cpp

@@ -820,9 +820,11 @@ js_ReportOutOfMemory(JSContext *cx)
         }
     }
 
-    if (onError)
+    if (onError) {
+        AutoScopedAssign<bool> ss(&cx->runtime->inOOMReport, true);
         onError(cx, msg, &report);
     }
+}
 
 void
 js_ReportOutOfScriptQuota(JSContext *maybecx)

js/src/jscntxt.h

@@ -705,6 +705,13 @@ struct JSRuntime {
     size_t               mjitDataSize;
 #endif
 
+    /*
+     * To ensure that cx->malloc does not cause a GC, we set this flag during
+     * OOM reporting (in js_ReportOutOfMemory). If a GC is requested while
+     * reporting the OOM, we ignore it.
+     */
+    bool                 inOOMReport;
+
     JSRuntime();
     ~JSRuntime();
 
@@ -715,7 +722,7 @@ struct JSRuntime {
 
     /*
      * Call the system malloc while checking for GC memory pressure and
-     * reporting OOM error when cx is not null.
+     * reporting OOM error when cx is not null. We will not GC from here.
      */
     void* malloc_(size_t bytes, JSContext *cx = NULL) {
         updateMallocCounter(bytes);
@@ -725,7 +732,7 @@ struct JSRuntime {
 
     /*
      * Call the system calloc while checking for GC memory pressure and
-     * reporting OOM error when cx is not null.
+     * reporting OOM error when cx is not null. We will not GC from here.
      */
     void* calloc_(size_t bytes, JSContext *cx = NULL) {
         updateMallocCounter(bytes);

js/src/jsgc.cpp

@@ -2683,6 +2683,10 @@ js_GC(JSContext *cx, JSCompartment *comp, JSGCInvocationKind gckind)
 {
     JSRuntime *rt = cx->runtime;
 
+    /* Don't GC while reporting an OOM. */
+    if (rt->inOOMReport)
+        return;
+
     /*
      * Don't collect garbage if the runtime isn't up, and cx is not the last
      * context in the runtime.  The last context must force a GC, and nothing

js/src/jstl.h

@@ -503,6 +503,25 @@ ImplicitCast(U &u)
     return t;
 }
 
+template<typename T>
+class AutoScopedAssign
+{
+  private:
+    JS_DECL_USE_GUARD_OBJECT_NOTIFIER
+    T *addr;
+    T old;
+
+  public:
+    AutoScopedAssign(T *addr, const T &value JS_GUARD_OBJECT_NOTIFIER_PARAM)
+        : addr(addr), old(*addr)
+    {
+        JS_GUARD_OBJECT_NOTIFIER_INIT;
+        *addr = value;
+    }
+
+    ~AutoScopedAssign() { *addr = old; }
+};
+
 } /* namespace js */
 
 #endif /* jstl_h_ */