Bug 1530687 - Check for recursion when delegating instanceof to a bound function target. r=anba

Differential Revision: https://phabricator.services.mozilla.com/D25409

--HG--
extra : moz-landing-system : lando

js/src/tests/non262/operators/instanceof-bound-function-recursion.js

@@ -0,0 +1,21 @@
+function f() {}
+
+var fn = f;
+for (var i = 0; i < 100000; ++i) {
+  fn = fn.bind();
+
+  // Ensure we don't fallback to @@hasInstance from %FunctionPrototype%.
+  Object.defineProperty(fn, Symbol.hasInstance, {
+    value: undefined, writable: true, enumerable: true, writable: true
+  });
+
+  // Prevent generating overlong names of the form "bound bound bound [...] f".
+  Object.defineProperty(fn, "name", {
+    value: "", writable: true, enumerable: true, writable: true
+  });
+}
+
+assertThrowsInstanceOf(
+  () => ({}) instanceof fn,
+  Error,
+  "detect runaway recursion delegating instanceof to bound function target");

js/src/vm/JSFunction.cpp

@@ -720,6 +720,9 @@ bool JS::OrdinaryHasInstance(JSContext* cx, HandleObject objArg, HandleValue v,
   /* Step 2. */
   if (obj->is<JSFunction>() && obj->isBoundFunction()) {
     /* Steps 2a-b. */
+    if (!CheckRecursionLimit(cx)) {
+      return false;
+    }
     obj = obj->as<JSFunction>().getBoundFunctionTarget();
     return InstanceofOperator(cx, obj, v, bp);
   }