Bug 1215058 - Fix various OOM handling issues related to off-thread compilation r=jandem

js/src/jit-test/tests/gc/oomInOffTheadCompile.js

@@ -0,0 +1,15 @@
+if (!('oomTest' in this) || helperThreadCount() === 0)
+    quit();
+
+oomTest(() => {
+    offThreadCompileScript(
+        `
+        function f(x) {
+            if (x == 0)
+                return "foobar";
+            return 1 + f(x - 1);
+        }
+        f(5);
+        `);
+    runOffThreadScript();
+});

js/src/jit/CodeGenerator.cpp

@@ -8008,7 +8008,11 @@ CodeGenerator::link(JSContext* cx, CompilerConstraintList* constraints)
     // will trickle to jit::Compile() and return Method_Skipped.
     uint32_t warmUpCount = script->getWarmUpCount();
     RecompileInfo recompileInfo;
-    if (!FinishCompilation(cx, script, constraints, &recompileInfo))
+    bool isValid;
+    if (!FinishCompilation(cx, script, constraints, &recompileInfo, &isValid))
+        return false;
+
+    if (!isValid)
         return true;
 
     // IonMonkey could have inferred better type information during

js/src/jit/CompactBuffer.h

@@ -125,7 +125,8 @@ class CompactBufferWriter
     }
     void writeByteAt(uint32_t pos, uint32_t byte) {
         MOZ_ASSERT(byte <= 0xFF);
-        buffer_[pos] = byte;
+        if (!oom())
+            buffer_[pos] = byte;
     }
     void writeUnsigned(uint32_t value) {
         do {
@@ -178,9 +179,11 @@ class CompactBufferWriter
         return buffer_.length();
     }
     uint8_t* buffer() {
+        MOZ_ASSERT(!oom());
         return &buffer_[0];
     }
     const uint8_t* buffer() const {
+        MOZ_ASSERT(!oom());
         return &buffer_[0];
     }
     bool oom() const {

js/src/jit/Snapshots.cpp

@@ -323,8 +323,7 @@ RValueAllocation::read(CompactBufferReader& reader)
 }
 
 void
-RValueAllocation::writePayload(CompactBufferWriter& writer, PayloadType type,
-                               Payload p)
+RValueAllocation::writePayload(CompactBufferWriter& writer, PayloadType type, Payload p)
 {
     switch (type) {
       case PAYLOAD_NONE:
@@ -348,10 +347,12 @@ RValueAllocation::writePayload(CompactBufferWriter& writer, PayloadType type,
       case PAYLOAD_PACKED_TAG: {
         // This code assumes that the PACKED_TAG payload is following the
         // writeByte of the mode.
-        MOZ_ASSERT(writer.length());
-        uint8_t* mode = writer.buffer() + (writer.length() - 1);
-        MOZ_ASSERT((*mode & PACKED_TAG_MASK) == 0 && (p.type & ~PACKED_TAG_MASK) == 0);
-        *mode = *mode | p.type;
+        if (!writer.oom()) {
+            MOZ_ASSERT(writer.length());
+            uint8_t* mode = writer.buffer() + (writer.length() - 1);
+            MOZ_ASSERT((*mode & PACKED_TAG_MASK) == 0 && (p.type & ~PACKED_TAG_MASK) == 0);
+            *mode = *mode | p.type;
+        }
         break;
       }
     }

js/src/jit/arm/CodeGenerator-arm.cpp

@@ -160,8 +160,8 @@ CodeGeneratorARM::bailoutFrom(Label* label, LSnapshot* snapshot)
     if (masm.bailed())
         return;
 
-    MOZ_ASSERT(label->used());
-    MOZ_ASSERT(!label->bound());
+    MOZ_ASSERT_IF(!masm.oom(), label->used());
+    MOZ_ASSERT_IF(!masm.oom(), !label->bound());
 
     encode(snapshot);
 

js/src/jit/mips-shared/CodeGenerator-mips-shared.cpp

@@ -171,8 +171,8 @@ CodeGeneratorMIPSShared::bailoutFrom(Label* label, LSnapshot* snapshot)
     if (masm.bailed())
         return;
 
-    MOZ_ASSERT(label->used());
-    MOZ_ASSERT(!label->bound());
+    MOZ_ASSERT_IF(!masm.oom(), label->used());
+    MOZ_ASSERT_IF(!masm.oom(), !label->bound());
 
     encode(snapshot);
 

js/src/jit/shared/IonAssemblerBufferWithConstantPools.h

@@ -518,7 +518,7 @@ struct AssemblerBufferWithConstantPools : public AssemblerBuffer<SliceSize, Inst
     size_t size() const {
         // Return the current actual size of the buffer. This is only accurate
         // if there are no pending pool entries to dump, check.
-        MOZ_ASSERT(pool_.numEntries() == 0);
+        MOZ_ASSERT_IF(!this->oom(), pool_.numEntries() == 0);
         return sizeExcludingCurrentPool();
     }
 
@@ -771,7 +771,7 @@ struct AssemblerBufferWithConstantPools : public AssemblerBuffer<SliceSize, Inst
         // state into the BufferSlice.
         Pool** tmp = &perforatedSlice->pool;
         *tmp = static_cast<Pool*>(this->lifoAlloc_.alloc(sizeof(Pool)));
-        if (tmp == nullptr) {
+        if (!*tmp) {
             this->fail_oom();
             return;
         }

js/src/vm/HelperThreads.cpp

@@ -404,16 +404,18 @@ js::StartOffThreadParseScript(JSContext* cx, const ReadOnlyCompileOptions& optio
 
     if (OffThreadParsingMustWaitForGC(cx->runtime())) {
         AutoLockHelperThreadState lock;
-        if (!HelperThreadState().parseWaitingOnGC().append(task.get()))
+        if (!HelperThreadState().parseWaitingOnGC().append(task.get())) {
+            ReportOutOfMemory(cx);
             return false;
+        }
     } else {
-        task->activate(cx->runtime());
-
         AutoLockHelperThreadState lock;
-
-        if (!HelperThreadState().parseWorklist().append(task.get()))
+        if (!HelperThreadState().parseWorklist().append(task.get())) {
+            ReportOutOfMemory(cx);
             return false;
+        }
 
+        task->activate(cx->runtime());
         HelperThreadState().notifyOne(GlobalHelperThreadState::PRODUCER);
     }
 
@@ -1066,16 +1068,21 @@ GlobalHelperThreadState::finishParseTask(JSContext* maybecx, JSRuntime* rt, void
     if (cx->isExceptionPending())
         return nullptr;
 
-    if (script) {
-        // The Debugger only needs to be told about the topmost script that was compiled.
-        Debugger::onNewScript(cx, script);
-
-        // Update the compressed source table with the result. This is normally
-        // called by setCompressedSource when compilation occurs on the main thread.
-        if (script->scriptSource()->hasCompressedSource())
-            script->scriptSource()->updateCompressedSourceSet(rt);
+    if (!script) {
+        // No error was reported, but no script produced. Assume we hit out of
+        // memory.
+        ReportOutOfMemory(cx);
+        return nullptr;
     }
 
+    // The Debugger only needs to be told about the topmost script that was compiled.
+    Debugger::onNewScript(cx, script);
+
+    // Update the compressed source table with the result. This is normally
+    // called by setCompressedSource when compilation occurs on the main thread.
+    if (script->scriptSource()->hasCompressedSource())
+        script->scriptSource()->updateCompressedSourceSet(rt);
+
     return script;
 }
 

js/src/vm/TypeInference.cpp

@@ -1368,7 +1368,7 @@ class TypeConstraintFreezeStack : public TypeConstraint
 
 bool
 js::FinishCompilation(JSContext* cx, HandleScript script, CompilerConstraintList* constraints,
-                      RecompileInfo* precompileInfo)
+                      RecompileInfo* precompileInfo, bool* isValidOut)
 {
     if (constraints->failed())
         return false;
@@ -1452,9 +1452,11 @@ js::FinishCompilation(JSContext* cx, HandleScript script, CompilerConstraintList
     if (!succeeded || types.compilerOutputs->back().pendingInvalidation()) {
         types.compilerOutputs->back().invalidate();
         script->resetWarmUpCounter();
-        return false;
+        *isValidOut = false;
+        return true;
     }
 
+    *isValidOut = true;
     return true;
 }
 

js/src/vm/TypeInference.h

@@ -1064,11 +1064,11 @@ FillBytecodeTypeMap(JSScript* script, uint32_t* bytecodeMap);
 class RecompileInfo;
 
 // Allocate a CompilerOutput for a finished compilation and generate the type
-// constraints for the compilation. Returns whether the type constraints
-// still hold.
+// constraints for the compilation. Sets |isValidOut| based on whether the type
+// constraints still hold.
 bool
 FinishCompilation(JSContext* cx, HandleScript script, CompilerConstraintList* constraints,
-                  RecompileInfo* precompileInfo);
+                  RecompileInfo* precompileInfo, bool* isValidOut);
 
 // Reset any CompilerOutput present for a script.
 void