Bug 1289392: Check for detached ArrayBuffers with inline or cross-compartment in TypedArray.prototype.sort. r=evilpie

js/src/builtin/TypedArray.js

@@ -1113,8 +1113,7 @@ function TypedArraySort(comparefn) {
     var buffer;
     if (isTypedArray) {
         buffer = GetAttachedArrayBuffer(obj);
-    }
-    else {
+    } else {
         buffer = callFunction(CallTypedArrayMethodIfWrapped, obj, obj, "GetAttachedArrayBuffer");
     }
 
@@ -1153,13 +1152,23 @@ function TypedArraySort(comparefn) {
         // Step a.
         var v = wrappedCompareFn(x, y);
         // Step b.
+        if (buffer === null) {
+            // A typed array previously using inline storage may acquire a
+            // buffer, so we must check with the source.
+            if (isTypedArray) {
+                buffer = GetAttachedArrayBuffer(obj);
+            } else {
+                buffer = callFunction(CallTypedArrayMethodIfWrapped, obj, obj,
+                                      "GetAttachedArrayBuffer");
+            }
+        }
         var bufferDetached;
         if (isTypedArray) {
             bufferDetached = IsDetachedBuffer(buffer);
         } else {
-            // This is totally cheating and only works because we know `this`
+            // This is totally cheating and only works because we know `obj`
             // and `buffer` are same-compartment".
-            bufferDetached = callFunction(CallTypedArrayMethodIfWrapped, this,
+            bufferDetached = callFunction(CallTypedArrayMethodIfWrapped, obj,
                                           buffer, "IsDetachedBuffer");
         }
         if (bufferDetached)

js/src/tests/ecma_6/TypedArray/sort_errors.js

@@ -8,6 +8,44 @@ if (typeof detachArrayBuffer === "function") {
     }, TypeError);
 }
 
+// Ensure detachment check works when buffer is detached in comparator.
+if (typeof detachArrayBuffer === "function") {
+    let detached = false;
+    let ta = new Int32Array(3);
+    assertThrowsInstanceOf(() => {
+        ta.sort(function(a, b) {
+            assertEq(detached, false);
+            detached = true;
+            detachArrayBuffer(ta.buffer);
+            return a - b;
+        });
+    }, TypeError);
+}
+
+// Ensure detachment check doesn't choke on wrapped typed array.
+if (typeof newGlobal === "function") {
+    let ta = new Int32Array(3);
+    let otherGlobal = newGlobal();
+    otherGlobal.Int32Array.prototype.sort.call(ta, function(a, b) {
+        return a - b;
+    });
+}
+
+// Ensure detachment check works for wrapped typed arrays.
+if (typeof newGlobal === "function" && typeof detachArrayBuffer === "function") {
+    let detached = false;
+    let ta = new Int32Array(3);
+    let otherGlobal = newGlobal();
+    assertThrowsInstanceOf(() => {
+        otherGlobal.Int32Array.prototype.sort.call(ta, function(a,b) {
+            assertEq(detached, false);
+            detached = true;
+            detachArrayBuffer(ta.buffer);
+            return a - b;
+        });
+    }, TypeError);
+}
+
 // Ensure that TypedArray.prototype.sort will not sort non-TypedArrays
 assertThrowsInstanceOf(() => {
     let array = [4, 3, 2, 1];