Bug 1155033 - Handle cyclic or deep ObjectGroup tracing chains during cycle collection, r=terrence.

js/public/HeapAPI.h

@@ -187,6 +187,7 @@ class JS_FRIEND_API(GCCellPtr)
     bool isString() const { return kind() == JSTRACE_STRING; }
     bool isSymbol() const { return kind() == JSTRACE_SYMBOL; }
     bool isShape() const { return kind() == JSTRACE_SHAPE; }
+    bool isObjectGroup() const { return kind() == JSTRACE_OBJECT_GROUP; }
 
     // Conversions to more specific types must match the kind. Access to
     // further refined types is not allowed directly from a GCCellPtr.

js/public/TracingAPI.h

@@ -42,6 +42,9 @@ enum JSGCTraceKind
     // Shape details are exposed through JS_TraceShapeCycleCollectorChildren.
     JSTRACE_SHAPE = 0x04,
 
+    // ObjectGroup details are exposed through JS_TraceObjectGroupCycleCollectorChildren.
+    JSTRACE_OBJECT_GROUP = 0x05,
+
     // The kind associated with a nullptr.
     JSTRACE_NULL = 0x06,
 
@@ -52,7 +55,6 @@ enum JSGCTraceKind
     JSTRACE_BASE_SHAPE = 0x0F,
     JSTRACE_JITCODE = 0x1F,
     JSTRACE_LAZY_SCRIPT = 0x2F,
-    JSTRACE_OBJECT_GROUP = 0x3F,
 
     JSTRACE_LAST = JSTRACE_OBJECT_GROUP
 };

js/src/gc/Marking.cpp

@@ -1108,6 +1108,83 @@ gc::MarkCycleCollectorChildren(JSTracer* trc, Shape* shape)
     } while (shape);
 }
 
+void
+TraceObjectGroupCycleCollectorChildrenCallback(JS::CallbackTracer* trc,
+                                               void** thingp, JSGCTraceKind kind);
+
+// Object groups can point to other object groups via an UnboxedLayout or the
+// the original unboxed group link. There can potentially be deep or cyclic
+// chains of such groups to trace through without going through a thing that
+// participates in cycle collection. These need to be handled iteratively to
+// avoid blowing the stack when running the cycle collector's callback tracer.
+struct ObjectGroupCycleCollectorTracer : public JS::CallbackTracer
+{
+    explicit ObjectGroupCycleCollectorTracer(JS::CallbackTracer* innerTracer)
+        : JS::CallbackTracer(innerTracer->runtime(),
+                             TraceObjectGroupCycleCollectorChildrenCallback,
+                             DoNotTraceWeakMaps),
+          innerTracer(innerTracer)
+    {}
+
+    JS::CallbackTracer* innerTracer;
+    Vector<ObjectGroup*, 4, SystemAllocPolicy> seen, worklist;
+};
+
+void
+TraceObjectGroupCycleCollectorChildrenCallback(JS::CallbackTracer* trcArg,
+                                               void** thingp, JSGCTraceKind kind)
+{
+    ObjectGroupCycleCollectorTracer* trc = static_cast<ObjectGroupCycleCollectorTracer*>(trcArg);
+    JS::GCCellPtr thing(*thingp, kind);
+
+    if (thing.isObject() || thing.isScript()) {
+        // Invoke the inner cycle collector callback on this child. It will not
+        // recurse back into TraceChildren.
+        trc->innerTracer->invoke(thingp, kind);
+        return;
+    }
+
+    if (thing.isObjectGroup()) {
+        // If this group is required to be in an ObjectGroup chain, trace it
+        // via the provided worklist rather than continuing to recurse.
+        ObjectGroup* group = static_cast<ObjectGroup*>(thing.asCell());
+        if (group->maybeUnboxedLayout()) {
+            for (size_t i = 0; i < trc->seen.length(); i++) {
+                if (trc->seen[i] == group)
+                    return;
+            }
+            if (trc->seen.append(group) && trc->worklist.append(group)) {
+                return;
+            } else {
+                // If append fails, keep tracing normally. The worst that will
+                // happen is we end up overrecursing.
+            }
+        }
+    }
+
+    TraceChildren(trc, thing.asCell(), thing.kind());
+}
+
+void
+gc::MarkCycleCollectorChildren(JSTracer* trc, ObjectGroup* group)
+{
+    MOZ_ASSERT(trc->isCallbackTracer());
+
+    // Early return if this group is not required to be in an ObjectGroup chain.
+    if (!group->maybeUnboxedLayout()) {
+        TraceChildren(trc, group, JSTRACE_OBJECT_GROUP);
+        return;
+    }
+
+    ObjectGroupCycleCollectorTracer groupTracer(trc->asCallbackTracer());
+    TraceChildren(&groupTracer, group, JSTRACE_OBJECT_GROUP);
+
+    while (!groupTracer.worklist.empty()) {
+        ObjectGroup* innerGroup = groupTracer.worklist.popCopy();
+        TraceChildren(&groupTracer, innerGroup, JSTRACE_OBJECT_GROUP);
+    }
+}
+
 static void
 ScanObjectGroup(GCMarker* gcmarker, ObjectGroup* group)
 {

js/src/gc/Marking.h

@@ -92,12 +92,13 @@ MarkObjectSlots(JSTracer* trc, NativeObject* obj, uint32_t start, uint32_t nslot
 /*** Special Cases ***/
 
 /*
- * Trace through the shape and any shapes it contains to mark
- * non-shape children. This is exposed to the JS API as
- * JS_TraceShapeCycleCollectorChildren.
+ * Trace through a shape or group iteratively during cycle collection to avoid
+ * deep or infinite recursion.
  */
 void
 MarkCycleCollectorChildren(JSTracer* trc, Shape* shape);
+void
+MarkCycleCollectorChildren(JSTracer* trc, ObjectGroup* group);
 
 void
 PushArena(GCMarker* gcmarker, ArenaHeader* aheader);

js/src/jsfriendapi.cpp

@@ -217,6 +217,13 @@ JS_TraceShapeCycleCollectorChildren(JSTracer* trc, JS::GCCellPtr shape)
     MarkCycleCollectorChildren(trc, static_cast<Shape*>(shape.asCell()));
 }
 
+JS_FRIEND_API(void)
+JS_TraceObjectGroupCycleCollectorChildren(JSTracer* trc, JS::GCCellPtr group)
+{
+    MOZ_ASSERT(group.isObjectGroup());
+    MarkCycleCollectorChildren(trc, static_cast<ObjectGroup*>(group.asCell()));
+}
+
 static bool
 DefineHelpProperty(JSContext* cx, HandleObject obj, const char* prop, const char* value)
 {

js/src/jsfriendapi.h

@@ -92,12 +92,13 @@ extern JS_FRIEND_API(bool)
 JS_IsDeadWrapper(JSObject* obj);
 
 /*
- * Used by the cycle collector to trace through the shape and all
- * shapes it reaches, marking all non-shape children found in the
- * process. Uses bounded stack space.
+ * Used by the cycle collector to trace through a shape or object group and
+ * all cycle-participating data it reaches, using bounded stack space.
  */
 extern JS_FRIEND_API(void)
 JS_TraceShapeCycleCollectorChildren(JSTracer* trc, JS::GCCellPtr shape);
+extern JS_FRIEND_API(void)
+JS_TraceObjectGroupCycleCollectorChildren(JSTracer* trc, JS::GCCellPtr group);
 
 enum {
     JS_TELEMETRY_GC_REASON,

xpcom/base/CycleCollectedJSRuntime.cpp

@@ -399,9 +399,8 @@ NoteJSChild(JS::CallbackTracer* aTrc, JS::GCCellPtr aThing)
    * This function needs to be careful to avoid stack overflow. Normally, when
    * AddToCCKind is true, the recursion terminates immediately as we just add
    * |thing| to the CC graph. So overflow is only possible when there are long
-   * chains of non-AddToCCKind GC things. Currently, this only can happen via
-   * shape parent pointers. The special JSTRACE_SHAPE case below handles
-   * parent pointers iteratively, rather than recursively, to avoid overflow.
+   * or cyclic chains of non-AddToCCKind GC things. Places where this can occur
+   * use special APIs to handle such chains iteratively.
    */
   if (AddToCCKind(aThing.kind())) {
     if (MOZ_UNLIKELY(tracer->mCb.WantDebugInfo())) {
@@ -415,7 +414,14 @@ NoteJSChild(JS::CallbackTracer* aTrc, JS::GCCellPtr aThing)
       tracer->mCb.NoteJSScript(aThing.toScript());
     }
   } else if (aThing.isShape()) {
+    // The maximum depth of traversal when tracing a Shape is unbounded, due to
+    // the parent pointers on the shape.
     JS_TraceShapeCycleCollectorChildren(aTrc, aThing);
+  } else if (aThing.isObjectGroup()) {
+    // The maximum depth of traversal when tracing an ObjectGroup is unbounded,
+    // due to information attached to the groups which can lead other groups to
+    // be traced.
+    JS_TraceObjectGroupCycleCollectorChildren(aTrc, aThing);
   } else if (!aThing.isString()) {
     JS_TraceChildren(aTrc, aThing.asCell(), aThing.kind());
   }