Bug 699565 - Part 1 - for-of loop basics. r=Waldo.

js/src/frontend/Parser.cpp

@@ -1867,6 +1867,7 @@ Parser::statements()
     tc->blockNode = saveBlock;
 
     pn->pn_pos.end = tokenStream.currentToken().pos.end;
+    JS_ASSERT(pn->pn_pos.begin <= pn->pn_pos.end);
     return pn;
 }
 
@@ -3136,6 +3137,23 @@ Parser::switchStatement()
     return pn;
 }
 
+bool
+Parser::matchInOrOf(bool *isForOfp)
+{
+    if (tokenStream.matchToken(TOK_IN)) {
+        *isForOfp = false;
+        return true;
+    }
+    if (tokenStream.matchToken(TOK_NAME)) {
+        if (tokenStream.currentToken().name() == context->runtime->atomState.ofAtom) {
+            *isForOfp = true;
+            return true;
+        }
+        tokenStream.ungetToken();
+    }
+    return false;
+}
+
 ParseNode *
 Parser::forStatement()
 {
@@ -3241,18 +3259,27 @@ Parser::forStatement()
     ParseNode *forHead;     /* initialized by both branches. */
     StmtInfo letStmt;       /* used if blockObj != NULL. */
     ParseNode *pn2, *pn3;   /* forHead->pn_kid1 and pn_kid2. */
-    if (pn1 && tokenStream.matchToken(TOK_IN)) {
+    bool forOf;
+    if (pn1 && matchInOrOf(&forOf)) {
         /*
-         * Parse the rest of the for/in head.
+         * Parse the rest of the for/in or for/of head.
          *
-         * Here pn1 is everything to the left of 'in'. At the end of this block,
-         * pn1 is a decl or NULL, pn2 is the assignment target that receives the
-         * enumeration value each iteration, and pn3 is the rhs of 'in'.
+         * Here pn1 is everything to the left of 'in' or 'of'. At the end of
+         * this block, pn1 is a decl or NULL, pn2 is the assignment target that
+         * receives the enumeration value each iteration, and pn3 is the rhs of
+         * 'in'.
          */
-        pn->pn_iflags |= JSITER_ENUMERATE;
         forStmt.type = STMT_FOR_IN_LOOP;
 
-        /* Check that the left side of the 'in' is valid. */
+        /* Set pn_iflags and rule out invalid combinations. */
+        if (forOf && pn->pn_iflags != 0) {
+            JS_ASSERT(pn->pn_iflags == JSITER_FOREACH);
+            reportErrorNumber(NULL, JSREPORT_ERROR, JSMSG_BAD_FOR_EACH_LOOP);
+            return NULL;
+        }
+        pn->pn_iflags |= (forOf ? JSITER_FOR_OF : JSITER_ENUMERATE);
+
+        /* Check that the left side of the 'in' or 'of' is valid. */
         if (forDecl
             ? (pn1->pn_count > 1 || pn1->isOp(JSOP_DEFCONST)
 #if JS_HAS_DESTRUCTURING
@@ -4323,7 +4350,9 @@ Parser::variables(ParseNodeKind kind, StaticBlockObject *blockObj, VarContext va
 
             if (!CheckDestructuring(context, &data, pn2, tc))
                 return NULL;
-            if ((tc->flags & TCF_IN_FOR_INIT) && tokenStream.peekToken() == TOK_IN) {
+            bool ignored;
+            if ((tc->flags & TCF_IN_FOR_INIT) && matchInOrOf(&ignored)) {
+                tokenStream.ungetToken();
                 pn->append(pn2);
                 continue;
             }
@@ -5373,7 +5402,7 @@ Parser::comprehensionTail(ParseNode *kid, uintN blockid, bool isGenexp,
         /*
          * FOR node is binary, left is loop control and right is body.  Use
          * index to count each block-local let-variable on the left-hand side
-         * of the IN.
+         * of the in/of.
          */
         pn2 = BinaryNode::create(PNK_FOR, tc);
         if (!pn2)
@@ -5427,7 +5456,20 @@ Parser::comprehensionTail(ParseNode *kid, uintN blockid, bool isGenexp,
             return NULL;
         }
 
-        MUST_MATCH_TOKEN(TOK_IN, JSMSG_IN_AFTER_FOR_NAME);
+        bool forOf;
+        if (!matchInOrOf(&forOf)) {
+            reportErrorNumber(NULL, JSREPORT_ERROR, JSMSG_IN_AFTER_FOR_NAME);
+            return NULL;
+        }
+        if (forOf) {
+            if (pn2->pn_iflags != JSITER_ENUMERATE) {
+                JS_ASSERT(pn2->pn_iflags == (JSITER_FOREACH | JSITER_ENUMERATE));
+                reportErrorNumber(NULL, JSREPORT_ERROR, JSMSG_BAD_FOR_EACH_LOOP);
+                return NULL;
+            }
+            pn2->pn_iflags = JSITER_FOR_OF;
+        }
+
         ParseNode *pn4 = expr();
         if (!pn4)
             return NULL;

js/src/frontend/Parser.h

@@ -280,6 +280,7 @@ struct Parser : private AutoGCRooter
 #endif /* JS_HAS_XML_SUPPORT */
 
     bool setAssignmentLhsOps(ParseNode *pn, JSOp op);
+    bool matchInOrOf(bool *isForOfp);
 };
 
 inline bool

js/src/jit-test/tests/for-of/decompiler.js

@@ -0,0 +1,36 @@
+// The decompiler correctly handles for-of loops.
+
+function tokens(code) {
+    var arr = [];
+    var s = code.replace(/\w+|[^\s]/g, function (tok) { arr.push(tok); return ""; });
+    assertEq(s.trim(), "", "tokens() should find all tokens in code: " + uneval(code));
+    return arr;
+}
+
+function test(code) {
+    var before = "function f() { " + code + " }";
+    var after = eval("(" + before + ")").toString();
+    assertEq(tokens(before).join(" "), tokens(after).join(" "), "decompiler failed to round-trip");
+}
+
+// statements
+test("for (a of b) { f(a); }");
+test("for (a of b) { f(a); g(a); }");
+
+// for-of with "in" operator nearby
+test("for (a of b in c ? c : c.items()) { f(a); }");
+
+// destructuring
+test("for ([a, b] of c) { a.m(b); }");
+
+// for-let-of
+test("for (let a of b) { f(a); }");
+test("for (let [a, b] of c) { a.m(b); }");
+
+// array comprehensions
+test("return [a for (a of b)];");
+test("return [[b, a] for ([a, b] of c.items())];");
+
+// generator expressions
+test("return (a for (a of b));");
+

js/src/jit-test/tests/for-of/generators-1.js

@@ -0,0 +1,11 @@
+// for-of works with generators.
+
+function range(n) {
+    for (var i = 0; i < n; i++)
+        yield i;
+}
+
+var s = '';
+for (var a of range(4))
+    s += a;
+assertEq(s, '0123');

js/src/jit-test/tests/for-of/generators-2.js

@@ -0,0 +1,15 @@
+// Generator-iterators are consumed the first time they are iterated.
+
+function range(n) {
+    for (var i = 0; i < n; i++)
+        yield i;
+}
+
+var r = range(10);
+var i = 0;
+for (var x in r)
+    assertEq(x, i++);
+assertEq(i, 10);
+for (var y in r)
+    throw "FAIL";
+assertEq(y, undefined);

js/src/jit-test/tests/for-of/generators-3.js

@@ -0,0 +1,18 @@
+// Nested for-of loops can use the same generator-iterator.
+
+function range(n) {
+    for (var i = 0; i < n; i++)
+        yield i;
+}
+
+var r = range(10);
+for (var a of r)
+    for (var b of r)
+        for (var c of r)
+            for (var d of r)
+                ;
+
+assertEq(a, 0);
+assertEq(b, 1);
+assertEq(c, 2);
+assertEq(d, 9);

js/src/jit-test/tests/for-of/generators-4.js

@@ -0,0 +1,15 @@
+// for-of can iterate over generator-iterators produced by generator-expressions.
+
+function g() {
+    yield 1;
+    yield 2;
+}
+
+var it = g();
+for (var i = 0; i < 10; i++) {
+    let prev = it;
+    it = (k + 1 for (k of prev));
+}
+
+var arr = [v for (v of it)];
+assertEq(arr.join(), "11,12");

js/src/jit-test/tests/for-of/generators-5.js

@@ -0,0 +1,18 @@
+// Breaking out of a for-of loop over a generator-iterator closes the iterator.
+
+function range(n) {
+    for (var i = 0; i < n; i++)
+        yield i;
+}
+
+var r = range(10);
+var s = '';
+for (var x of r) {
+    s += x;
+    if (x == 4)
+        break;
+}
+s += '/';
+for (var y of r)
+    s += y;
+assertEq(s, '01234/');

js/src/jit-test/tests/for-of/generators-6.js

@@ -0,0 +1,43 @@
+// Named break closes the right generator-iterators.
+
+function g() {
+    for (;;)
+        yield 1;
+}
+
+function isClosed(it) {
+    try {
+        it.next();
+        return false;
+    } catch (exc) {
+        if (exc !== StopIteration)
+            throw exc;
+        return true;
+    }
+}
+
+var a = g(), b = g(), c = g(), d = g(), e = g(), f = g();
+
+for (var aa of a) {
+    b_: for (var bb of b) {
+        c_: for (var cc of c) {
+            d_: for (var dd of d) {
+                e_: for (var ee of e) {
+                    for (var ff of f)
+                        break c_;
+                }
+            }
+        }
+        assertEq(isClosed(a), false);
+        assertEq(isClosed(b), false);
+        assertEq(isClosed(c), true);
+        assertEq(isClosed(d), true);
+        assertEq(isClosed(e), true);
+        assertEq(isClosed(f), true);
+        break b_;
+    }
+    assertEq(isClosed(a), false);
+    assertEq(isClosed(b), true);
+    break;
+}
+assertEq(isClosed(a), true);

js/src/jit-test/tests/for-of/non-iterable.js

@@ -0,0 +1,21 @@
+// Iterating over non-iterable values throws a TypeError.
+
+load(libdir + "asserts.js");
+
+var misc = [
+    {}, {x: 1}, Math, isNaN,
+    Object.create(null),
+    Object.create(Array.prototype),
+    null, undefined,
+    true, 0, 3.1416, "", "ponies",
+    new Boolean(true), new Number(0), new String("ponies")];
+
+for (var i = 0; i < misc.length; i++) {
+    let v = misc[i];
+    var testfn = function () {
+        for (var _ of v)
+            throw 'FAIL';
+        throw 'BAD';
+    };
+    assertThrowsInstanceOf(testfn, TypeError);
+}

js/src/jit-test/tests/for-of/proxy-1.js

@@ -0,0 +1,28 @@
+// Basic for-of test with Proxy.
+
+function iter(arr) {
+    var i = 0;
+    return {
+        next: function () {
+            if (i < arr.length)
+                return arr[i++];
+            throw StopIteration;
+        }
+    };
+}
+
+function iterableProxy(arr) {
+    return Proxy.create({iterate: function () { return iter(arr); }});
+}
+
+var s = '';
+var arr = ['a', 'b', 'c', 'd'];
+var p = iterableProxy(arr);
+
+// Test the same proxy twice. Its iterate method should be called each time.
+for (var i = 0; i < 2; i++) {
+    var j = 0;
+    for (var x of p)
+        assertEq(x, arr[j++]);
+    assertEq(j, arr.length);
+}

js/src/jit-test/tests/for-of/proxy-2.js

@@ -0,0 +1,12 @@
+// Basic for-of test with Proxy whose iterate method is a generator.
+
+var arr = ['a', 'b', 'c', 'd'];
+var proxy = Proxy.create({
+    iterate: function () {
+        for (var i = 0; i < arr.length; i++)
+            yield arr[i];
+    }
+});
+
+for (var i = 0; i < 2; i++)
+    assertEq([v for (v of proxy)].join(","), "a,b,c,d");

js/src/jit-test/tests/for-of/proxy-3.js

@@ -0,0 +1,6 @@
+// An exception thrown from an iterate trap is propagated.
+
+load(libdir + "asserts.js");
+
+var p = Proxy.create({iterate: function () { throw "fit"; }});
+assertThrowsValue(function () { for (var v of p) {} }, "fit");

js/src/jit-test/tests/for-of/proxy-4.js

@@ -0,0 +1,10 @@
+// for-of on a fixed (non-trapping) proxy does not call the iterate trap.
+
+load(libdir + "asserts.js");
+
+var p = Proxy.create({
+    iterate: function () { throw "FAIL"; },
+    fix: function () { return {}; }
+});
+Object.preventExtensions(p);
+assertThrowsInstanceOf(function () { for (var v of p) {} }, TypeError);

js/src/jit-test/tests/for-of/syntax-1.js

@@ -0,0 +1,22 @@
+// We correctly reject bogus for-of loop syntax.
+
+load(libdir + "asserts.js");
+
+function assertSyntaxError(code) {
+    assertThrowsInstanceOf(function () { Function(code); }, SyntaxError, "Function:" + code);
+    assertThrowsInstanceOf(function () { eval(code); }, SyntaxError, "eval:" + code);
+    var ieval = eval;
+    assertThrowsInstanceOf(function () { ieval(code); }, SyntaxError, "indirect eval:" + code);
+}
+
+function test(badForHead) {
+    assertSyntaxError(badForHead + " {}");  // apply directly to forHead
+    assertSyntaxError("[0 " + badForHead + "];");
+}
+
+var a, b, c;
+test("for (a in b of c)");
+test("for each (a of b)");
+test("for (a of b of c)");
+test("for (let (a = 1) a of b)");
+test("for (let {a: 1} of b)");

js/src/jit-test/tests/for-of/syntax-2.js

@@ -0,0 +1,9 @@
+// "of" is not a keyword.
+
+var of;
+
+Function("var of;");
+
+let (of = 12) {}
+
+function of(of) {}

js/src/js.msg

@@ -294,7 +294,7 @@ MSG_DEF(JSMSG_WRONG_CONSTRUCTOR,      207, 1, JSEXN_TYPEERR, "wrong constructor
 MSG_DEF(JSMSG_BAD_GENERATOR_RETURN,   208, 1, JSEXN_TYPEERR, "generator function {0} returns a value")
 MSG_DEF(JSMSG_BAD_ANON_GENERATOR_RETURN, 209, 0, JSEXN_TYPEERR, "anonymous generator function returns a value")
 MSG_DEF(JSMSG_NAME_AFTER_FOR_PAREN,   210, 0, JSEXN_SYNTAXERR, "missing name after for (")
-MSG_DEF(JSMSG_IN_AFTER_FOR_NAME,      211, 0, JSEXN_SYNTAXERR, "missing in after for")
+MSG_DEF(JSMSG_IN_AFTER_FOR_NAME,      211, 0, JSEXN_SYNTAXERR, "missing 'in' or 'of' after for")
 MSG_DEF(JSMSG_BAD_TRAP_RETURN_VALUE,  212, 2, JSEXN_TYPEERR,"trap {1} for {0} returned a primitive value")
 MSG_DEF(JSMSG_KEYWORD_NOT_NS,         213, 0, JSEXN_SYNTAXERR, "keyword is used as namespace")
 MSG_DEF(JSMSG_BAD_GENERATOR_YIELD,    214, 1, JSEXN_TYPEERR, "yield from closing generator {0}")
@@ -374,3 +374,4 @@ MSG_DEF(JSMSG_CANT_WATCH_PROP,        287, 0, JSEXN_TYPEERR, "properties whose n
 MSG_DEF(JSMSG_CSP_BLOCKED_EVAL,       288, 0, JSEXN_ERR, "call to eval() blocked by CSP")
 MSG_DEF(JSMSG_DEBUG_NO_SCOPE_OBJECT,  289, 0, JSEXN_TYPEERR, "declarative Environments don't have binding objects")
 MSG_DEF(JSMSG_EMPTY_CONSEQUENT,       290, 0, JSEXN_SYNTAXERR, "mistyped ; after conditional?")
+MSG_DEF(JSMSG_NOT_ITERABLE,           291, 1, JSEXN_TYPEERR, "{0} is not iterable")

js/src/jsapi.h

@@ -3438,7 +3438,12 @@ struct JSClass {
 #define JSCLASS_HIGH_FLAGS_SHIFT        (JSCLASS_RESERVED_SLOTS_SHIFT +       \
                                          JSCLASS_RESERVED_SLOTS_WIDTH)
 
-#define JSCLASS_INTERNAL_FLAG1          (1<<(JSCLASS_HIGH_FLAGS_SHIFT+0))
+/*
+ * Call the iteratorObject hook only to iterate over contents (for-of), not to
+ * enumerate properties (for-in, for-each, Object.keys, etc.)
+ */
+#define JSCLASS_FOR_OF_ITERATION        (1<<(JSCLASS_HIGH_FLAGS_SHIFT+0))
+
 #define JSCLASS_IS_ANONYMOUS            (1<<(JSCLASS_HIGH_FLAGS_SHIFT+1))
 #define JSCLASS_IS_GLOBAL               (1<<(JSCLASS_HIGH_FLAGS_SHIFT+2))
 #define JSCLASS_INTERNAL_FLAG2          (1<<(JSCLASS_HIGH_FLAGS_SHIFT+3))

js/src/jsatom.cpp

@@ -154,6 +154,7 @@ const char *const js_common_atom_names[] = {
     js_noSuchMethod_str,        /* noSuchMethodAtom             */
     "[object Null]",            /* objectNullAtom               */
     "[object Undefined]",       /* objectUndefinedAtom          */
+    "of",                       /* ofAtom                       */
     js_proto_str,               /* protoAtom                    */
     js_set_str,                 /* setAtom                      */
     js_source_str,              /* sourceAtom                   */

js/src/jsatom.h

@@ -314,6 +314,7 @@ struct JSAtomState
     js::PropertyName    *noSuchMethodAtom;
     js::PropertyName    *objectNullAtom;
     js::PropertyName    *objectUndefinedAtom;
+    js::PropertyName    *ofAtom;
     js::PropertyName    *protoAtom;
     js::PropertyName    *setAtom;
     js::PropertyName    *sourceAtom;

js/src/jsfriendapi.h

@@ -517,6 +517,7 @@ IsObjectInContextCompartment(const JSObject *obj, const JSContext *cx);
 #define JSITER_KEYVALUE   0x4   /* destructuring for-in wants [key, value] */
 #define JSITER_OWNONLY    0x8   /* iterate over obj's own properties only */
 #define JSITER_HIDDEN     0x10  /* also enumerate non-enumerable properties */
+#define JSITER_FOR_OF     0x20  /* harmony for-of loop */
 
 inline uintptr_t
 GetContextStackLimit(const JSContext *cx)

js/src/jsiter.cpp

@@ -423,6 +423,16 @@ GetCustomIterator(JSContext *cx, JSObject *obj, uintN flags, Value *vp)
 {
     JS_CHECK_RECURSION(cx, return false);
 
+    /*
+     * for-of iteration does not fall back on __iterator__ or property
+     * enumeration. This is more conservative than the current proposed spec.
+     */
+    if (flags == JSITER_FOR_OF) {
+        js_ReportValueErrorFlags(cx, JSREPORT_ERROR, JSMSG_NOT_ITERABLE,
+                                 JSDVG_SEARCH_STACK, ObjectValue(*obj), NULL, NULL, NULL);
+        return false;
+    }
+
     /* Check whether we have a valid __iterator__ method. */
     JSAtom *atom = cx->runtime->atomState.iteratorAtom;
     if (!js_GetMethod(cx, obj, ATOM_TO_JSID(atom), JSGET_NO_METHOD_BARRIER, vp))
@@ -658,12 +668,22 @@ GetIterator(JSContext *cx, JSObject *obj, uintN flags, Value *vp)
     if (obj) {
         /* Enumerate Iterator.prototype directly. */
         if (JSIteratorOp op = obj->getClass()->ext.iteratorObject) {
-            JSObject *iterobj = op(cx, obj, !(flags & JSITER_FOREACH));
-            if (!iterobj)
-                return false;
-            vp->setObject(*iterobj);
-            types::MarkIteratorUnknown(cx);
-            return true;
+            /*
+             * Arrays and other classes representing iterable collections have
+             * the JSCLASS_FOR_OF_ITERATION flag. This flag means that the
+             * object responds to all other kinds of enumeration (for-in,
+             * for-each, Object.keys, Object.getOwnPropertyNames, etc.) in the
+             * default way, ignoring the hook. The hook is used only when
+             * iterating in the style of a for-of loop.
+             */
+            if (!(obj->getClass()->flags & JSCLASS_FOR_OF_ITERATION) || flags == JSITER_FOR_OF) {
+                JSObject *iterobj = op(cx, obj, !(flags & (JSITER_FOREACH | JSITER_FOR_OF)));
+                if (!iterobj)
+                    return false;
+                vp->setObject(*iterobj);
+                types::MarkIteratorUnknown(cx);
+                return true;
+            }
         }
 
         if (keysOnly) {
@@ -1260,7 +1280,7 @@ Class js::GeneratorClass = {
         NULL,                /* equality       */
         NULL,                /* outerObject    */
         NULL,                /* innerObject    */
-        iterator_iterator,   
+        iterator_iterator,
         NULL                 /* unused */
     }
 };

js/src/jsopcode.cpp

@@ -2712,6 +2712,7 @@ Decompile(SprintStack *ss, jsbytecode *pc, intN nb)
     saveop = JSOP_NOP;
     sn = NULL;
     rval = NULL;
+    bool forOf = false;
 #if JS_HAS_XML_SUPPORT
     foreach = inXML = quoteAttr = JS_FALSE;
 #endif
@@ -3854,6 +3855,7 @@ Decompile(SprintStack *ss, jsbytecode *pc, intN nb)
                 break;
 
               case JSOP_ITER:
+                forOf = (GET_UINT8(pc) == JSITER_FOR_OF);
                 foreach = (GET_UINT8(pc) & (JSITER_FOREACH | JSITER_KEYVALUE)) ==
                           JSITER_FOREACH;
                 todo = -2;
@@ -3925,9 +3927,11 @@ Decompile(SprintStack *ss, jsbytecode *pc, intN nb)
                         rval = POP_STR();
                         if (ss->top >= 1 && ss->opcodes[ss->top - 1] == JSOP_FORLOCAL) {
                             ss->sprinter.setOffset(ss->offsets[ss->top] - PAREN_SLOP);
-                            if (Sprint(&ss->sprinter, " %s (%s in %s)",
+                            if (Sprint(&ss->sprinter, " %s (%s %s %s)",
                                        foreach ? js_for_each_str : js_for_str,
-                                       lval, rval) < 0) {
+                                       lval,
+                                       forOf ? "of" : "in",
+                                       rval) < 0) {
                                 return NULL;
                             }
 
@@ -3939,9 +3943,11 @@ Decompile(SprintStack *ss, jsbytecode *pc, intN nb)
                              */
                             todo = ss->offsets[ss->top - 1];
                         } else {
-                            todo = Sprint(&ss->sprinter, " %s (%s in %s)",
+                            todo = Sprint(&ss->sprinter, " %s (%s %s %s)",
                                           foreach ? js_for_each_str : js_for_str,
-                                          lval, rval);
+                                          lval,
+                                          forOf ? "of" : "in",
+                                          rval);
                         }
                         if (todo < 0 || !PushOff(ss, todo, JSOP_FORLOCAL))
                             return NULL;
@@ -3953,9 +3959,12 @@ Decompile(SprintStack *ss, jsbytecode *pc, intN nb)
                          */
                         rval = GetStr(ss, ss->top - 1);
                         xval = VarPrefix(js_GetSrcNote(jp->script, pc + next));
-                        js_printf(jp, "\t%s (%s%s in %s) {\n",
+                        js_printf(jp, "\t%s (%s%s %s %s) {\n",
                                   foreach ? js_for_each_str : js_for_str,
-                                  xval, lval, rval);
+                                  xval,
+                                  lval,
+                                  forOf ? "of" : "in",
+                                  rval);
                         jp->indent += 4;
                         DECOMPILE_CODE(pc + next + JSOP_POP_LENGTH, cond - next - JSOP_POP_LENGTH);
                         jp->indent -= 4;