Bug 1288944 - Baldr: fuse the WasmTableObject InstanceVector into the Table's array (r=bbouvier)

MozReview-Commit-ID: BdP4hd2WX2S

js/src/asmjs/WasmInstance.cpp

@@ -350,7 +350,7 @@ Instance::Instance(JSContext* cx,
     }
 
     for (size_t i = 0; i < tables_.length(); i++)
-        *addressOfTableBase(i) = tables_[i]->array();
+        *addressOfTableBase(i) = tables_[i]->base();
 }
 
 bool
@@ -407,6 +407,9 @@ Instance::tracePrivate(JSTracer* trc)
     for (const FuncImport& fi : metadata().funcImports)
         TraceNullableEdge(trc, &funcImportTls(fi).obj, "wasm import");
 
+    for (const SharedTable& table : tables_)
+        table->trace(trc);
+
     TraceNullableEdge(trc, &memory_, "wasm buffer");
 }
 
@@ -501,6 +504,12 @@ ReadCustomDoubleNaNObject(JSContext* cx, HandleValue v, double* ret)
     return true;
 }
 
+WasmInstanceObject*
+Instance::object() const
+{
+    return object_;
+}
+
 bool
 Instance::callExport(JSContext* cx, uint32_t funcIndex, CallArgs args)
 {
@@ -767,7 +776,7 @@ Instance::ensureProfilingState(JSContext* cx, bool newProfilingEnabled)
         // can contain elements from multiple instances.
         MOZ_ASSERT(metadata().kind == ModuleKind::AsmJS);
 
-        void** array = table->array();
+        void** array = table->internalArray();
         uint32_t length = table->length();
         for (size_t i = 0; i < length; i++)
             UpdateEntry(*code_, newProfilingEnabled, &array[i]);

js/src/asmjs/WasmInstance.h

@@ -43,10 +43,6 @@ class Instance
     const UniqueCode                     code_;
     GCPtrWasmMemoryObject                memory_;
     SharedTableVector                    tables_;
-
-    // Thread-local data for code running in this instance.
-    // When threading is supported, we need a TlsData object per thread per
-    // instance.
     TlsData                              tlsData_;
 
     // Internal helpers:
@@ -91,12 +87,13 @@ class Instance
     const SharedTableVector& tables() const { return tables_; }
     SharedMem<uint8_t*> memoryBase() const;
     size_t memoryLength() const;
+    TlsData& tlsData() { return tlsData_; }
 
     // This method returns a pointer to the GC object that owns this Instance.
     // Instances may be reached via weak edges (e.g., Compartment::instances_)
     // so this perform a read-barrier on the returned object.
 
-    WasmInstanceObject* object() const { return object_; }
+    WasmInstanceObject* object() const;
 
     // Execute the given export given the JS call arguments, storing the return
     // value in args.rval.

js/src/asmjs/WasmJS.cpp

@@ -849,16 +849,14 @@ WasmTableObject::finalize(FreeOp* fop, JSObject* obj)
     WasmTableObject& tableObj = obj->as<WasmTableObject>();
     if (!tableObj.isNewborn())
         tableObj.table().Release();
-    if (tableObj.initialized())
-        fop->delete_(&tableObj.instanceVector());
 }
 
 /* static */ void
 WasmTableObject::trace(JSTracer* trc, JSObject* obj)
 {
     WasmTableObject& tableObj = obj->as<WasmTableObject>();
-    if (tableObj.initialized())
+    if (!tableObj.isNewborn())
-        tableObj.instanceVector().trace(trc);
+        tableObj.table().trace(trc);
 }
 
 /* static */ WasmTableObject*
@@ -886,40 +884,9 @@ WasmTableObject::create(JSContext* cx, uint32_t length)
     obj->initReservedSlot(TABLE_SLOT, PrivateValue(table.forget().take()));
 
     MOZ_ASSERT(!obj->isNewborn());
-    MOZ_ASSERT(!obj->initialized());
     return obj;
 }
 
-bool
-WasmTableObject::initialized() const
-{
-    return !getReservedSlot(INSTANCE_VECTOR_SLOT).isUndefined();
-}
-
-bool
-WasmTableObject::init(JSContext* cx, HandleWasmInstanceObject instanceObj)
-{
-    MOZ_ASSERT(!initialized());
-    MOZ_ASSERT(!table().initialized());
-
-    // Ensure initialization is atomic so that the table is never left in an
-    // inconsistent state (where the Table is initialized but the
-    // WasmTableObject is not).
-
-    auto instanceVector = MakeUnique<InstanceVector>();
-    if (!instanceVector || !instanceVector->appendN(instanceObj.get(), table().length())) {
-        ReportOutOfMemory(cx);
-        return false;
-    }
-
-    initReservedSlot(INSTANCE_VECTOR_SLOT, PrivateValue(instanceVector.release()));
-    table().init(instanceObj->instance().codeSegment());
-
-    MOZ_ASSERT(initialized());
-    MOZ_ASSERT(table().initialized());
-    return true;
-}
-
 /* static */ bool
 WasmTableObject::construct(JSContext* cx, unsigned argc, Value* vp)
 {
@@ -1037,16 +1004,14 @@ WasmTableObject::getImpl(JSContext* cx, const CallArgs& args)
     uint32_t index = uint32_t(indexDbl);
     MOZ_ASSERT(double(index) == indexDbl);
 
-    if (!tableObj->initialized()) {
+    if (!table.initialized()) {
         args.rval().setNull();
         return true;
     }
 
-    const InstanceVector& instanceVector = tableObj->instanceVector();
+    ExternalTableElem& elem = table.externalArray()[index];
-    MOZ_ASSERT(instanceVector.length() == table.length());
+    Instance& instance = *elem.tls->instance;
-
+    const CodeRange* codeRange = instance.code().lookupRange(elem.code);
-    RootedWasmInstanceObject instanceObj(cx, instanceVector[index]);
-    const CodeRange* codeRange = instanceObj->instance().code().lookupRange(table.array()[index]);
 
     // A non-function code range means the bad-indirect-call stub, so a null element.
     if (!codeRange || !codeRange->isFunction()) {
@@ -1054,6 +1019,7 @@ WasmTableObject::getImpl(JSContext* cx, const CallArgs& args)
         return true;
     }
 
+    RootedWasmInstanceObject instanceObj(cx, instance.object());
     RootedFunction fun(cx);
     if (!instanceObj->getExportedFunction(cx, instanceObj, codeRange->funcIndex(), &fun))
         return false;
@@ -1073,7 +1039,7 @@ WasmTableObject::get(JSContext* cx, unsigned argc, Value* vp)
 WasmTableObject::setImpl(JSContext* cx, const CallArgs& args)
 {
     RootedWasmTableObject tableObj(cx, &args.thisv().toObject().as<WasmTableObject>());
-    const Table& table = tableObj->table();
+    Table& table = tableObj->table();
 
     if (!args.requireAtLeast(cx, "set", 2))
         return false;
@@ -1096,20 +1062,15 @@ WasmTableObject::setImpl(JSContext* cx, const CallArgs& args)
         return false;
     }
 
-    if (!tableObj->initialized()) {
+    if (!table.initialized()) {
         if (!value) {
             args.rval().setUndefined();
             return true;
         }
 
-        RootedWasmInstanceObject instanceObj(cx, ExportedFunctionToInstanceObject(value));
+        table.init(ExportedFunctionToInstance(value));
-        if (!tableObj->init(cx, instanceObj))
-            return false;
     }
 
-    const InstanceVector& instanceVector = tableObj->instanceVector();
-    MOZ_ASSERT(instanceVector.length() == table.length());
-
     if (value) {
         RootedWasmInstanceObject instanceObj(cx, ExportedFunctionToInstanceObject(value));
         uint32_t funcIndex = ExportedFunctionToIndex(value);
@@ -1120,15 +1081,15 @@ WasmTableObject::setImpl(JSContext* cx, const CallArgs& args)
         MOZ_ASSERT(value == f);
 #endif
 
-        if (!tableObj->setInstance(cx, index, instanceObj))
-            return false;
-
         Instance& instance = instanceObj->instance();
         const FuncExport& funcExport = instance.metadata().lookupFuncExport(funcIndex);
         const CodeRange& codeRange = instance.metadata().codeRanges[funcExport.codeRangeIndex()];
-        table.array()[index] = instance.codeSegment().base() + codeRange.funcTableEntry();
+        void* code = instance.codeSegment().base() + codeRange.funcTableEntry();
+
+        if (!table.set(cx, index, code, instance))
+            return false;
     } else {
-        table.array()[index] = instanceVector[index]->instance().codeSegment().badIndirectCallCode();
+        table.setNull(index);
     }
 
     args.rval().setUndefined();
@@ -1155,28 +1116,6 @@ WasmTableObject::table() const
     return *(Table*)getReservedSlot(TABLE_SLOT).toPrivate();
 }
 
-WasmTableObject::InstanceVector&
-WasmTableObject::instanceVector() const
-{
-    MOZ_ASSERT(initialized());
-    return *(InstanceVector*)getReservedSlot(INSTANCE_VECTOR_SLOT).toPrivate();
-}
-
-bool
-WasmTableObject::setInstance(JSContext* cx, uint32_t index, HandleWasmInstanceObject instanceObj)
-{
-    MOZ_ASSERT(initialized());
-    MOZ_ASSERT(instanceObj->instance().codeSegment().containsCodePC(table().array()[index]));
-
-    if (instanceVector()[index] != instanceObj) {
-        JS_ReportError(cx, "cross-module Table import NYI");
-        return false;
-    }
-
-    instanceVector()[index] = instanceObj;
-    return true;
-}
-
 // ============================================================================
 // WebAssembly class and static methods
 

js/src/asmjs/WasmJS.h

@@ -180,7 +180,6 @@ class WasmMemoryObject : public NativeObject
 class WasmTableObject : public NativeObject
 {
     static const unsigned TABLE_SLOT = 0;
-    static const unsigned INSTANCE_VECTOR_SLOT = 1;
     static const ClassOps classOps_;
     bool isNewborn() const;
     static void finalize(FreeOp* fop, JSObject* obj);
@@ -192,29 +191,18 @@ class WasmTableObject : public NativeObject
     static bool setImpl(JSContext* cx, const CallArgs& args);
     static bool set(JSContext* cx, unsigned argc, Value* vp);
 
-    // InstanceVector has the same length as the Table and assigns, to each
-    // element, the instance of the exported function stored in that element.
-    using InstanceVector = GCVector<HeapPtr<WasmInstanceObject*>, 0, SystemAllocPolicy>;
-    InstanceVector& instanceVector() const;
-
   public:
-    static const unsigned RESERVED_SLOTS = 2;
+    static const unsigned RESERVED_SLOTS = 1;
     static const Class class_;
     static const JSPropertySpec properties[];
     static const JSFunctionSpec methods[];
     static bool construct(JSContext*, unsigned, Value*);
 
+    // Note that, after creation, a WasmTableObject's table() is not initialized
+    // and must be initialized before use.
+
     static WasmTableObject* create(JSContext* cx, uint32_t length);
-    bool initialized() const;
-    bool init(JSContext* cx, HandleWasmInstanceObject instanceObj);
-
-    // As a global invariant, any time an element of tableObj->table() is
-    // updated to a new exported function, table->setInstance() must be called
-    // to update the instance of that new exported function in the instance
-    // vector.
-
     wasm::Table& table() const;
-    bool setInstance(JSContext* cx, uint32_t index, HandleWasmInstanceObject instanceObj);
 };
 
 } // namespace js

js/src/asmjs/WasmModule.cpp

@@ -415,15 +415,10 @@ Module::initElems(JSContext* cx, HandleWasmInstanceObject instanceObj,
     Instance& instance = instanceObj->instance();
     const SharedTableVector& tables = instance.tables();
 
-    // Initialize tables that have a WasmTableObject first, so that this
+    // Ensure all tables are initialized before storing into them.
-    // initialization can be done atomically.
-    if (tableObj && !tableObj->initialized() && !tableObj->init(cx, instanceObj))
-        return false;
-
-    // Initialize all remaining Tables that do not have objects.
     for (const SharedTable& table : tables) {
         if (!table->initialized())
-            table->init(instance.code().segment());
+            table->init(instance);
     }
 
     // Now that all tables have been initialized, write elements.
@@ -462,14 +457,6 @@ Module::initElems(JSContext* cx, HandleWasmInstanceObject instanceObj,
             return false;
         }
 
-        if (tableObj) {
-            MOZ_ASSERT(seg.tableIndex == 0);
-            for (uint32_t i = 0; i < seg.elems.length(); i++) {
-                if (!tableObj->setInstance(cx, offset + i, instanceObj))
-                    return false;
-            }
-        }
-
         // If profiling is already enabled in the wasm::Compartment, the new
         // instance must use the profiling entry for typed functions instead of
         // the default nonProfilingEntry.
@@ -477,10 +464,12 @@ Module::initElems(JSContext* cx, HandleWasmInstanceObject instanceObj,
 
         uint8_t* codeBase = instance.codeBase();
         for (uint32_t i = 0; i < seg.elems.length(); i++) {
-            void* callee = codeBase + seg.elems[i];
+            void* code = codeBase + seg.elems[i];
             if (useProfilingEntry)
-                callee = codeBase + instance.code().lookupRange(callee)->funcProfilingEntry();
+                code = codeBase + instance.code().lookupRange(code)->funcProfilingEntry();
-            table.array()[offset + i] = callee;
+
+            if (!table.set(cx, offset + i, code, instance))
+                return false;
         }
 
         prevEnd = offset + seg.elems.length();

js/src/asmjs/WasmTable.cpp

@@ -20,6 +20,8 @@
 
 #include "jscntxt.h"
 
+#include "asmjs/WasmInstance.h"
+
 using namespace js;
 using namespace js::wasm;
 
@@ -30,27 +32,97 @@ Table::create(JSContext* cx, const TableDesc& desc)
     if (!table)
         return nullptr;
 
-    table->array_.reset(cx->pod_calloc<void*>(desc.initial));
+    // The raw element type of a Table depends on whether it is external: an
-    if (!table->array_)
+    // external table can contain functions from multiple instances and thus
+    // must store an additional instance pointer in each element.
+    void* array;
+    if (desc.external)
+        array = cx->pod_calloc<ExternalTableElem>(desc.initial);
+    else
+        array = cx->pod_calloc<void*>(desc.initial);
+    if (!array)
         return nullptr;
 
+    table->array_.reset((uint8_t*)array);
     table->kind_ = desc.kind;
     table->length_ = desc.initial;
     table->initialized_ = false;
+    table->external_ = desc.external;
     return table;
 }
 
 void
-Table::init(const CodeSegment& codeSegment)
+Table::init(Instance& instance)
 {
     MOZ_ASSERT(!initialized());
+    initialized_ = true;
 
-    for (uint32_t i = 0; i < length_; i++) {
+    void* code = instance.codeSegment().badIndirectCallCode();
-        MOZ_ASSERT(!array_.get()[i]);
+    if (external_) {
-        array_.get()[i] = codeSegment.badIndirectCallCode();
+        ExternalTableElem* array = externalArray();
+        TlsData* tls = &instance.tlsData();
+        for (uint32_t i = 0; i < length_; i++) {
+            array[i].code = code;
+            array[i].tls = tls;
+        }
+    } else {
+        void** array = internalArray();
+        for (uint32_t i = 0; i < length_; i++)
+            array[i] = code;
+    }
+}
+
+void
+Table::trace(JSTracer* trc)
+{
+    if (!initialized_ || !external_)
+        return;
+
+    ExternalTableElem* array = externalArray();
+    for (uint32_t i = 0; i < length_; i++)
+        array[i].tls->instance->trace(trc);
+}
+
+void**
+Table::internalArray() const
+{
+    MOZ_ASSERT(initialized_);
+    MOZ_ASSERT(!external_);
+    return (void**)array_.get();
+}
+
+ExternalTableElem*
+Table::externalArray() const
+{
+    MOZ_ASSERT(initialized_);
+    MOZ_ASSERT(external_);
+    return (ExternalTableElem*)array_.get();
+}
+
+bool
+Table::set(JSContext* cx, uint32_t index, void* code, Instance& instance)
+{
+    if (external_) {
+        ExternalTableElem& elem = externalArray()[index];
+        if (elem.tls->instance != &instance) {
+            JS_ReportError(cx, "cross-module Table import NYI");
+            return false;
+        }
+
+        elem.code = code;
+    } else {
+        internalArray()[index] = code;
     }
 
-    initialized_ = true;
+    return true;
+}
+
+void
+Table::setNull(uint32_t index)
+{
+    // Only external tables can set elements to null after initialization.
+    ExternalTableElem& elem = externalArray()[index];
+    elem.code = elem.tls->instance->codeSegment().badIndirectCallCode();
 }
 
 size_t

js/src/asmjs/WasmTable.h

@@ -30,24 +30,36 @@ namespace wasm {
 
 class Table : public ShareableBase<Table>
 {
+    UniquePtr<uint8_t[], JS::FreePolicy> array_;
     TableKind kind_;
-    UniquePtr<void*> array_;
     uint32_t length_;
     bool initialized_;
+    bool external_;
 
   public:
     static RefPtr<Table> create(JSContext* cx, const TableDesc& desc);
+    void trace(JSTracer* trc);
 
     // These accessors may be used before initialization.
 
+    bool external() const { return external_; }
     bool isTypedFunction() const { return kind_ == TableKind::TypedFunction; }
-    void** array() const { return array_.get(); }
     uint32_t length() const { return length_; }
+    uint8_t* base() const { return array_.get(); }
 
     // A Table must be initialized before any dependent instance can execute.
 
     bool initialized() const { return initialized_; }
-    void init(const CodeSegment& codeSegment);
+    void init(Instance& instance);
+
+    // After initialization, elements must be accessed. All updates must go
+    // through a set() function with the exception of (profiling) updates to the
+    // callee pointer that do not change which logical function is being called.
+
+    void** internalArray() const;
+    ExternalTableElem* externalArray() const;
+    bool set(JSContext* cx, uint32_t index, void* code, Instance& instance);
+    void setNull(uint32_t index);
 
     // about:memory reporting:
 

js/src/asmjs/WasmTypes.h

@@ -1141,6 +1141,10 @@ class CalleeDesc
         MOZ_ASSERT(which_ == WasmTable);
         return u.table.desc_.initial;
     }
+    bool wasmTableIsExternal() const {
+        MOZ_ASSERT(which_ == WasmTable);
+        return u.table.desc_.external;
+    }
     SigIdDesc wasmTableSigId() const {
         MOZ_ASSERT(which_ == WasmTable);
         return u.table.sigId_;
@@ -1220,6 +1224,24 @@ struct FuncImportTls
     static_assert(sizeof(GCPtrObject) == sizeof(void*), "for JIT access");
 };
 
+// When a table can be shared between instances (it is "external"), the internal
+// representation is an array of ExternalTableElem instead of just an array of
+// code pointers.
+
+struct ExternalTableElem
+{
+    // The code to call when calling this element. The table ABI is the system
+    // ABI with the additional ABI requirements that:
+    //  - WasmTlsReg and any pinned registers have been loaded appropriately
+    //  - if this is a heterogeneous table that requires a signature check,
+    //    WasmTableCallSigReg holds the signature id.
+    void* code;
+
+    // The pointer to the callee's instance's TlsData. This must be loaded into
+    // WasmTlsReg before calling 'code'.
+    TlsData* tls;
+};
+
 // Constants:
 
 // The WebAssembly spec hard-codes the virtual page size to be 64KiB and

js/src/jit/MacroAssembler.cpp

@@ -2756,6 +2756,8 @@ MacroAssembler::wasmCallIndirect(const wasm::CallSiteDesc& desc, const wasm::Cal
     Register index = WasmTableCallIndexReg;
 
     if (callee.which() == wasm::CalleeDesc::AsmJSTable) {
+        // asm.js tables require no signature check, have had their index masked
+        // into range and thus need no bounds check and cannot be external.
         loadWasmGlobalPtr(callee.tableGlobalDataOffset(), scratch);
         loadPtr(BaseIndex(scratch, index, ScalePointer), scratch);
         call(desc, scratch);
@@ -2782,8 +2784,23 @@ MacroAssembler::wasmCallIndirect(const wasm::CallSiteDesc& desc, const wasm::Cal
              index, Imm32(callee.wasmTableLength()),
              wasm::JumpTarget::OutOfBounds);
 
+    // Load the base pointer of the table.
     loadWasmGlobalPtr(callee.tableGlobalDataOffset(), scratch);
-    loadPtr(BaseIndex(scratch, index, ScalePointer), scratch);
+
+    // Load the callee from the table.
+    if (callee.wasmTableIsExternal()) {
+        static_assert(sizeof(wasm::ExternalTableElem) == 8 || sizeof(wasm::ExternalTableElem) == 16,
+                      "elements of external tables are two words");
+        if (sizeof(wasm::ExternalTableElem) == 8) {
+            loadPtr(BaseIndex(scratch, index, TimesEight), scratch);
+        } else {
+            lshift32(Imm32(4), index);
+            loadPtr(BaseIndex(scratch, index, TimesOne), scratch);
+        }
+    } else {
+        loadPtr(BaseIndex(scratch, index, ScalePointer), scratch);
+    }
+
     call(desc, scratch);
 }