Bug 1763996: Handle NaN values in ComparatorNumeric. r=tcampbell

Normalising `NaN` comparator results to `0` was added in ES6, but the two
numeric comparator functions were never updated to include this change.

Differential Revision: https://phabricator.services.mozilla.com/D143337

devtools/client/webconsole/test/browser/browser_webconsole_console_table_post_alterations.js

@@ -16,7 +16,15 @@ add_task(async function() {
     content.wrappedJSObject.console.table(x);
     x.push("c");
     content.wrappedJSObject.console.table(x);
-    x.sort((a, b) => b - a);
+    x.sort((a, b) => {
+      if (a < b) {
+        return 1;
+      }
+      if (a > b) {
+        return -1;
+      }
+      return 0;
+    });
     content.wrappedJSObject.console.table(x);
   });
 

js/src/builtin/Array.cpp

@@ -13,6 +13,7 @@
 #include "mozilla/TextUtils.h"
 
 #include <algorithm>
+#include <cmath>
 #include <iterator>
 
 #include "jsfriendapi.h"
@@ -1769,14 +1770,14 @@ struct NumericElement {
 static bool ComparatorNumericLeftMinusRight(const NumericElement& a,
                                             const NumericElement& b,
                                             bool* lessOrEqualp) {
-  *lessOrEqualp = (a.dv <= b.dv);
+  *lessOrEqualp = std::isunordered(a.dv, b.dv) || (a.dv <= b.dv);
   return true;
 }
 
 static bool ComparatorNumericRightMinusLeft(const NumericElement& a,
                                             const NumericElement& b,
                                             bool* lessOrEqualp) {
-  *lessOrEqualp = (b.dv <= a.dv);
+  *lessOrEqualp = std::isunordered(a.dv, b.dv) || (b.dv <= a.dv);
   return true;
 }
 

js/src/tests/non262/Array/sort_native_string_nan.js

@@ -0,0 +1,16 @@
+
+var array = ["not-a-number", "also-not-a-number"];
+var copy = [...array];
+
+// The sort comparator must be exactly equal to the bytecode pattern:
+//
+// JSOp::GetArg 0/1
+// JSOp::GetArg 1/0
+// JSOp::Sub
+// JSOp::Return
+array.sort(function(a, b) { return a - b; });
+
+assertEqArray(array, copy);
+
+if (typeof reportCompare === 'function')
+  reportCompare(0, 0);