Attempt to fix the issues by disabling IM when we run into a corner case, rev 3, looks good on try (bug 805299, r=dvander)

js/src/ion/IonLinker.h

@@ -55,6 +55,8 @@ class Linker
                                      bytesNeeded - headerSize, pool);
         if (!code)
             return NULL;
+        if (masm.oom())
+            return fail(cx);
         code->copyFrom(masm);
         masm.link(code);
         return code;

js/src/ion/IonMacroAssembler.h

@@ -503,7 +503,7 @@ class MacroAssembler : public MacroAssemblerSpecific
     }
 
     void link(IonCode *code) {
-
+        JS_ASSERT(!oom());
         // If this code can transition to C++ code and witness a GC, then we need to store
         // the IonCode onto the stack in order to GC it correctly.  exitCodePatch should
         // be unset if the code never needed to push its IonCode*.

js/src/ion/arm/Assembler-arm.cpp

@@ -1637,6 +1637,10 @@ Assembler::as_b(BOffImm off, Condition c, bool isPatchable)
 BufferOffset
 Assembler::as_b(Label *l, Condition c, bool isPatchable)
 {
+    if (m_buffer.oom()) {
+        BufferOffset ret;
+        return ret;
+    }
     m_buffer.markNextAsBranch();
     if (l->bound()) {
         BufferOffset ret = as_nop();

js/src/ion/shared/IonAssemblerBuffer.h

@@ -77,7 +77,7 @@ struct BufferSlice : public InlineForwardListNode<BufferSlice<SliceSize> > {
 template<int SliceSize, class Inst>
 struct AssemblerBuffer {
   public:
-    AssemblerBuffer() : head(NULL), tail(NULL), m_oom(false), bufferSize(0) {}
+    AssemblerBuffer() : head(NULL), tail(NULL), m_bail(false), m_oom(false), bufferSize(0) {}
   protected:
     typedef BufferSlice<SliceSize> Slice;
     typedef AssemblerBuffer<SliceSize, Inst> AssemblerBuffer_;
@@ -85,6 +85,7 @@ struct AssemblerBuffer {
     Slice *tail;
   public:
     bool m_oom;
+    bool m_bail;
     // How much data has been added to the buffer thusfar.
     uint32_t bufferSize;
     uint32_t lastInstSize;
@@ -148,11 +149,17 @@ struct AssemblerBuffer {
         return size();
     }
     bool oom() const {
-        return m_oom;
+        return m_oom || m_bail;
+    }
+    bool bail() const {
+        return m_bail;
     }
     void fail_oom() {
         m_oom = true;
     }
+    void fail_bail() {
+        m_bail = true;
+    }
     Inst *getInst(BufferOffset off) {
         unsigned int local_off = off.getOffset();
         Slice *cur = NULL;

js/src/ion/shared/IonAssemblerBufferWithConstantPools.h

@@ -431,7 +431,7 @@ struct AssemblerBufferWithConstantPool : public AssemblerBuffer<SliceSize, Inst>
     }
 
     BufferOffset insertEntry(uint32_t instSize, uint8_t *inst, Pool *p, uint8_t *data, PoolEntry *pe = NULL) {
-        if (this->oom())
+        if (this->oom() && !this->bail())
             return BufferOffset();
         int token;
         if (p != NULL) {
@@ -451,6 +451,8 @@ struct AssemblerBufferWithConstantPool : public AssemblerBuffer<SliceSize, Inst>
         // now to get an instruction to write
         PoolEntry retPE;
         if (p != NULL) {
+            if (this->oom())
+                return BufferOffset();
             int poolId = p - pools;
             IonSpew(IonSpew_Pools, "[%d] Entry has token %d, offset ~%d", id, token, size());
             Asm::insertTokenIntoTag(instSize, inst, token);
@@ -496,6 +498,8 @@ struct AssemblerBufferWithConstantPool : public AssemblerBuffer<SliceSize, Inst>
                     IonSpew(IonSpew_Pools, "[%d]Inserting instruction(%d) caused a spill", id, size());
 
                 this->finishPool();
+                if (this->oom())
+                    return uint32_t(-1);
                 return this->insertEntryForwards(instSize, inst, p, data);
             }
             // when moving back to front, calculating the alignment is hard, just be
@@ -726,9 +730,9 @@ struct AssemblerBufferWithConstantPool : public AssemblerBuffer<SliceSize, Inst>
                     // the last pool, which means it cannot affect the alignment of any other
                     // Sub Pools.
                     IonSpew(IonSpew_Pools, "[%d]***Offset was still out of range!***", id, codeOffset - magicAlign);
-                    outcasts[poolIdx].append(iter->getOffset());
-                    memcpy(&outcastEntries[poolIdx][numSkips * p->immSize], &p->poolData[idx * p->immSize], p->immSize);
-                    numSkips++;
+                    IonSpew(IonSpew_Pools, "[%d] Too complicated; bailingp", id);
+                    this->fail_bail();
+                    return;
                 } else {
                     preservedEntries[idx] = true;
                 }