Bug 1290579 - Float32 RadixSort ignores typed array byte offsets. r=Waldo

--HG--
extra : rebase_source : d7c70654a3a2361b0fbfa1f7c85ef96015635c7b
extra : histedit_source : 5eeaff75d7701b2ed05f7a8b5894077295eb480a

js/src/builtin/Sorting.js

@@ -8,6 +8,8 @@
 
 // For sorting values with limited range; uint8 and int8.
 function CountingSort(array, len, signed, comparefn) {
+    assert(IsPossiblyWrappedTypedArray(array), "CountingSort works only with typed arrays.");
+
     // Determined by performance testing.
     if (len < 128) {
         QuickSort(array, len, comparefn);
@@ -109,6 +111,8 @@ function SortByColumn(array, len, aux, col, counts) {
 // Sorts integers and float32. |signed| is true for int16 and int32, |floating|
 // is true for float32.
 function RadixSort(array, len, buffer, nbytes, signed, floating, comparefn) {
+    assert(IsPossiblyWrappedTypedArray(array), "RadixSort works only with typed arrays.");
+
     // Determined by performance testing.
     if (len < 512) {
         QuickSort(array, len, comparefn);
@@ -131,7 +135,13 @@ function RadixSort(array, len, buffer, nbytes, signed, floating, comparefn) {
             assert(buffer !== null, "Attached data buffer should be reified");
         }
 
-        view = new Int32Array(buffer);
+        // |array| is a possibly cross-compartment wrapped typed array.
+        let offset = IsTypedArray(array)
+                     ? TypedArrayByteOffset(array)
+                     : callFunction(CallTypedArrayMethodIfWrapped, array, array,
+                                    "TypedArrayByteOffset");
+
+        view = new Int32Array(buffer, offset, len);
 
         // Flip sign bit for positive numbers; flip all bits for negative
         // numbers

js/src/tests/ecma_6/TypedArray/sort_byteoffset.js

@@ -0,0 +1,30 @@
+// Ensure that when sorting TypedArrays we don't
+// ignore byte offsets (bug 1290579).
+
+var sortFunctions = [Int32Array.prototype.sort];
+
+// Also test with cross-compartment wrapped typed arrays.
+if (typeof newGlobal === "function") {
+    var otherGlobal = newGlobal();
+    sortFunctions.push(newGlobal().Int32Array.prototype.sort);
+}
+
+// The bug manifests itself only with Float arrays,
+// but checking everything here just for sanity.
+
+for (var ctor of anyTypedArrayConstructors) {
+    var ab = new ArrayBuffer(1025 * ctor.BYTES_PER_ELEMENT);
+    var ta = new ctor(ab, ctor.BYTES_PER_ELEMENT, 1024);
+
+    // |testArray[0]| shouldn't be modified when sort() is called below.
+    var testArray = new ctor(ab, 0, 1);
+    testArray[0] = 1;
+
+    for (var sortFn of sortFunctions) {
+        sortFn.call(ta);
+        assertEq(testArray[0], 1);
+    }
+}
+
+if (typeof reportCompare === "function")
+    reportCompare(true, true);