Bug 1429028: Add a stack recursion check in the wasm::TextToBinary function; r=luke

MozReview-Commit-ID: HPXo0ARuKD6

--HG--
extra : rebase_source : 66be4206150f75be6aa941b6625ce18fa3efdd84

js/src/builtin/TestingFunctions.cpp

@@ -614,9 +614,11 @@ WasmTextToBinary(JSContext* cx, unsigned argc, Value* vp)
         }
     }
 
+    uintptr_t stackLimit = GetNativeStackLimit(cx);
+
     wasm::Bytes bytes;
     UniqueChars error;
-    if (!wasm::TextToBinary(twoByteChars.twoByteChars(), &bytes, &error)) {
+    if (!wasm::TextToBinary(twoByteChars.twoByteChars(), stackLimit, &bytes, &error)) {
         JS_ReportErrorNumberASCII(cx, GetErrorMessage, nullptr, JSMSG_WASM_TEXT_FAIL,
                                   error.get() ? error.get() : "out of memory");
         return false;

js/src/jit-test/tests/wasm/compiler-frame-depth.js

@@ -15,4 +15,12 @@ var code = `(module
  (export "run" 0)
 )`;
 
-wasmFullPass(code, Math.fround(13.37), {}, 13.37);
+try {
+    wasmFullPass(code, Math.fround(13.37), {}, 13.37);
+} catch (e) {
+    // ASAN will fail these tests because its stack frames are much bigger than
+    // usual ones and the parser will bail out during its recursive descent.
+    // Ignore those errors specifically.
+    assertEq(e.message.includes('out of memory'), true);
+    assertEq(getBuildConfiguration().asan, true);
+}

js/src/wasm/WasmTextToBinary.cpp

@@ -1665,12 +1665,15 @@ struct WasmParseContext
     LifoAlloc& lifo;
     UniqueChars* error;
     DtoaState* dtoaState;
+    uintptr_t stackLimit;
 
-    WasmParseContext(const char16_t* text, LifoAlloc& lifo, UniqueChars* error)
+    WasmParseContext(const char16_t* text, uintptr_t stackLimit, LifoAlloc& lifo,
+                     UniqueChars* error)
       : ts(text, error),
         lifo(lifo),
         error(error),
-        dtoaState(NewDtoaState())
+        dtoaState(NewDtoaState()),
+        stackLimit(stackLimit)
     {}
 
     ~WasmParseContext() {
@@ -2869,6 +2872,8 @@ ParseGrowMemory(WasmParseContext& c, bool inParens)
 static AstExpr*
 ParseExprBody(WasmParseContext& c, WasmToken token, bool inParens)
 {
+    if (!CheckRecursionLimitDontReport(c.stackLimit))
+        return nullptr;
     switch (token.kind()) {
       case WasmToken::Unreachable:
         return new(c.lifo) AstUnreachable;
@@ -3728,9 +3733,10 @@ ParseBinaryModule(WasmParseContext& c, AstModule* module)
 }
 
 static AstModule*
-ParseModule(const char16_t* text, LifoAlloc& lifo, UniqueChars* error, bool* binary)
+ParseModule(const char16_t* text, uintptr_t stackLimit, LifoAlloc& lifo, UniqueChars* error,
+            bool* binary)
 {
-    WasmParseContext c(text, lifo, error);
+    WasmParseContext c(text, stackLimit, lifo, error);
 
     *binary = false;
 
@@ -5436,12 +5442,12 @@ EncodeBinaryModule(const AstModule& module, Bytes* bytes)
 /*****************************************************************************/
 
 bool
-wasm::TextToBinary(const char16_t* text, Bytes* bytes, UniqueChars* error)
+wasm::TextToBinary(const char16_t* text, uintptr_t stackLimit, Bytes* bytes, UniqueChars* error)
 {
     LifoAlloc lifo(AST_LIFO_DEFAULT_CHUNK_SIZE);
 
     bool binary = false;
-    AstModule* module = ParseModule(text, lifo, error, &binary);
+    AstModule* module = ParseModule(text, stackLimit, lifo, error, &binary);
     if (!module)
         return false;
 

js/src/wasm/WasmTextToBinary.h

@@ -29,7 +29,7 @@ namespace wasm {
 // other than out-of-memory an error message string will be stored in 'error'.
 
 extern MOZ_MUST_USE bool
-TextToBinary(const char16_t* text, Bytes* bytes, UniqueChars* error);
+TextToBinary(const char16_t* text, uintptr_t stackLimit, Bytes* bytes, UniqueChars* error);
 
 } // namespace wasm
 } // namespace js