diff --git a/js/src/jit-test/tests/debug/bug1251919.js b/js/src/jit-test/tests/debug/bug1251919.js new file mode 100644 index 000000000000..188bfa39327a --- /dev/null +++ b/js/src/jit-test/tests/debug/bug1251919.js @@ -0,0 +1,13 @@ +// |jit-test| error: out of memory + +if (!('oomTest' in this)) + throw new Error("out of memory"); + +// jsfunfuzz-generated +fullcompartmentchecks(true); +// Adapted from randomly chosen test: js/src/jit-test/tests/debug/bug-1248162.js +var dbg = new Debugger; +dbg.onNewGlobalObject = function() {}; +oomTest(function() { + newGlobal(); +}) diff --git a/js/src/vm/Debugger.cpp b/js/src/vm/Debugger.cpp index 0ef4e9841a80..57d62caf181c 100644 --- a/js/src/vm/Debugger.cpp +++ b/js/src/vm/Debugger.cpp @@ -398,6 +398,14 @@ Debugger::slowPathCheckNoExecute(JSContext* cx, HandleScript script) return EnterDebuggeeNoExecute::reportIfFoundInStack(cx, script); } +static inline void +NukeDebuggerWrapper(NativeObject *wrapper) +{ + // In some OOM failure cases, we need to destroy the edge to the referent, + // to avoid trying to trace it during untimely collections. + wrapper->setPrivate(nullptr); +} + /*** Breakpoints *********************************************************************************/ @@ -934,14 +942,19 @@ Debugger::wrapEnvironment(JSContext* cx, Handle env, MutableHandleValue rv return false; envobj->setPrivateGCThing(env); envobj->setReservedSlot(JSSLOT_DEBUGENV_OWNER, ObjectValue(*object)); - if (!p.add(cx, environments, env, envobj)) + + if (!p.add(cx, environments, env, envobj)) { + NukeDebuggerWrapper(envobj); return false; + } CrossCompartmentKey key(CrossCompartmentKey::DebuggerEnvironment, object, env); if (!object->compartment()->putWrapper(cx, key, ObjectValue(*envobj))) { + NukeDebuggerWrapper(envobj); environments.remove(env); return false; } + } rval.setObject(*envobj); return true; @@ -976,12 +989,15 @@ Debugger::wrapDebuggeeValue(JSContext* cx, MutableHandleValue vp) dobj->setPrivateGCThing(obj); dobj->setReservedSlot(JSSLOT_DEBUGOBJECT_OWNER, ObjectValue(*object)); - if (!p.add(cx, objects, obj, dobj)) + if (!p.add(cx, objects, obj, dobj)) { + NukeDebuggerWrapper(dobj); return false; + } if (obj->compartment() != object->compartment()) { CrossCompartmentKey key(CrossCompartmentKey::DebuggerObject, object, obj); if (!object->compartment()->putWrapper(cx, key, ObjectValue(*dobj))) { + NukeDebuggerWrapper(dobj); objects.remove(obj); ReportOutOfMemory(cx); return false; @@ -5029,7 +5045,7 @@ class DebuggerScriptSetPrivateMatcher ReturnType match(Handle module) { obj_->setPrivateGCThing(module); } }; -JSObject* +NativeObject* Debugger::newDebuggerScript(JSContext* cx, Handle referent) { assertSameCompartment(cx, object.get()); @@ -5059,19 +5075,23 @@ Debugger::wrapVariantReferent(JSContext* cx, Map& map, CrossCompartmentKey::Kind DependentAddPtr p(cx, map, untaggedReferent); if (!p) { - JSObject* wrapper = newVariantWrapper(cx, referent); + NativeObject* wrapper = newVariantWrapper(cx, referent); if (!wrapper) return nullptr; - if (!p.add(cx, map, untaggedReferent, wrapper)) + if (!p.add(cx, map, untaggedReferent, wrapper)) { + NukeDebuggerWrapper(wrapper); return nullptr; + } CrossCompartmentKey key(keyKind, object, untaggedReferent); if (!object->compartment()->putWrapper(cx, key, ObjectValue(*wrapper))) { + NukeDebuggerWrapper(wrapper); map.remove(untaggedReferent); ReportOutOfMemory(cx); return nullptr; } + } return p->value(); @@ -6301,7 +6321,7 @@ class SetDebuggerSourcePrivateMatcher ReturnType match(Handle module) { obj_->setPrivateGCThing(module); } }; -JSObject* +NativeObject* Debugger::newDebuggerSource(JSContext* cx, Handle referent) { assertSameCompartment(cx, object.get()); diff --git a/js/src/vm/Debugger.h b/js/src/vm/Debugger.h index e833e084bb97..bcc5e6742b74 100644 --- a/js/src/vm/Debugger.h +++ b/js/src/vm/Debugger.h @@ -690,10 +690,10 @@ class Debugger : private mozilla::LinkedListElement JSTrapStatus fireNewGlobalObject(JSContext* cx, Handle global, MutableHandleValue vp); JSTrapStatus firePromiseHook(JSContext* cx, Hook hook, HandleObject promise, MutableHandleValue vp); - JSObject* newVariantWrapper(JSContext* cx, Handle referent) { + NativeObject* newVariantWrapper(JSContext* cx, Handle referent) { return newDebuggerScript(cx, referent); } - JSObject* newVariantWrapper(JSContext* cx, Handle referent) { + NativeObject* newVariantWrapper(JSContext* cx, Handle referent) { return newDebuggerSource(cx, referent); } @@ -715,13 +715,13 @@ class Debugger : private mozilla::LinkedListElement * Allocate and initialize a Debugger.Script instance whose referent is * |referent|. */ - JSObject* newDebuggerScript(JSContext* cx, Handle referent); + NativeObject* newDebuggerScript(JSContext* cx, Handle referent); /* * Allocate and initialize a Debugger.Source instance whose referent is * |referent|. */ - JSObject* newDebuggerSource(JSContext* cx, Handle referent); + NativeObject* newDebuggerSource(JSContext* cx, Handle referent); /* * Receive a "new script" event from the engine. A new script was compiled