Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-19 00:05:36 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1407314 - Improve tenuring performance by rearranging allocKindForTenure() methods and adding fast path for plain objects r=sfink

Browse Source
This commit is contained in:
Jon Coppeard 2017-10-17 09:40:50 +01:00
parent ebfa77072c
commit 2ff66da1a7
9 changed files with 221 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
js/src/builtin/TypedObject.cpp
View File

@ -24,6 +24,7 @@
#include "jsatominlines.h"
#include "jsobjinlines.h"
#include "gc/Nursery-inl.h"
#include "gc/StoreBuffer-inl.h"
#include "vm/NativeObject-inl.h"
#include "vm/Shape-inl.h"
@ -1244,7 +1245,7 @@ GlobalObject::initTypedObjectModule(JSContext* cx, Handle<GlobalObject*> global)
return false;
Rooted<TypedObjectModuleObject*> module(cx);
module = NewObjectWithGivenProto<TypedObjectModuleObject>(cx, objProto);
module = NewObjectWithGivenProto<TypedObjectModuleObject>(cx, objProto, SingletonObject);
if (!module)
return false;

161
js/src/gc/Marking.cpp
View File

@ -38,6 +38,7 @@
#include "jsobjinlines.h"
#include "gc/Nursery-inl.h"
#include "vm/NativeObject-inl.h"
#include "vm/String-inl.h"
#include "vm/UnboxedObject-inl.h"
@ -2668,8 +2669,18 @@ TenuringTracer::traverse(JSObject** objp)
// We only ever visit the internals of objects after moving them to tenured.
MOZ_ASSERT(!nursery().isInside(objp));
if (IsInsideNursery(*objp) && !nursery().getForwardedPointer(objp))
*objp = moveToTenured(*objp);
JSObject* obj = *objp;
if (!IsInsideNursery(obj) || nursery().getForwardedPointer(objp))
return;
// Take a fast path for tenuring a plain object which is by far the most
// common case.
if (obj->is<PlainObject>()) {
*objp = movePlainObjectToTenured(&obj->as<PlainObject>());
return;
}
*objp = moveToTenuredSlow(obj);
}
template <typename S>
@ -2831,58 +2842,6 @@ js::gc::StoreBuffer::ValueEdge::trace(TenuringTracer& mover) const
mover.traverse(edge);
}
/* Insert the given relocation entry into the list of things to visit. */
void
js::TenuringTracer::insertIntoFixupList(RelocationOverlay* entry) {
*tail = entry;
tail = &entry->nextRef();
*tail = nullptr;
}
JSObject*
js::TenuringTracer::moveToTenured(JSObject* src)
{
MOZ_ASSERT(IsInsideNursery(src));
MOZ_ASSERT(!src->zone()->usedByHelperThread());
AllocKind dstKind = src->allocKindForTenure(nursery());
Zone* zone = src->zone();
TenuredCell* t = zone->arenas.allocateFromFreeList(dstKind, Arena::thingSize(dstKind));
if (!t) {
AutoEnterOOMUnsafeRegion oomUnsafe;
t = runtime()->gc.refillFreeListInGC(zone, dstKind);
if (!t)
oomUnsafe.crash(ChunkSize, "Failed to allocate object while tenuring.");
}
JSObject* dst = reinterpret_cast<JSObject*>(t);
tenuredSize += moveObjectToTenured(dst, src, dstKind);
RelocationOverlay* overlay = RelocationOverlay::fromCell(src);
overlay->forwardTo(dst);
insertIntoFixupList(overlay);
TracePromoteToTenured(src, dst);
return dst;
}
void
js::Nursery::collectToFixedPoint(TenuringTracer& mover, TenureCountCache& tenureCounts)
{
for (RelocationOverlay* p = mover.head; p; p = p->next()) {
JSObject* obj = static_cast<JSObject*>(p->forwardingAddress());
mover.traceObject(obj);
TenureCount& entry = tenureCounts.findEntry(obj->groupRaw());
if (entry.group == obj->groupRaw()) {
entry.count++;
} else if (!entry.group) {
entry.group = obj->groupRaw();
entry.count = 1;
}
}
}
struct TenuringFunctor
{
template <typename T>
@ -2942,11 +2901,39 @@ OffsetToChunkEnd(void* p)
}
#endif
size_t
js::TenuringTracer::moveObjectToTenured(JSObject* dst, JSObject* src, AllocKind dstKind)
/* Insert the given relocation entry into the list of things to visit. */
inline void
js::TenuringTracer::insertIntoFixupList(RelocationOverlay* entry) {
*tail = entry;
tail = &entry->nextRef();
*tail = nullptr;
}
template <typename T>
inline T*
js::TenuringTracer::allocTenured(Zone* zone, AllocKind kind) {
TenuredCell* t = zone->arenas.allocateFromFreeList(kind, Arena::thingSize(kind));
if (!t) {
AutoEnterOOMUnsafeRegion oomUnsafe;
t = runtime()->gc.refillFreeListInGC(zone, kind);
if (!t)
oomUnsafe.crash(ChunkSize, "Failed to allocate object while tenuring.");
}
return static_cast<T*>(static_cast<Cell*>(t));
}
JSObject*
js::TenuringTracer::moveToTenuredSlow(JSObject* src)
{
MOZ_ASSERT(IsInsideNursery(src));
MOZ_ASSERT(!src->zone()->usedByHelperThread());
MOZ_ASSERT(!src->is<PlainObject>());
AllocKind dstKind = src->allocKindForTenure(nursery());
auto dst = allocTenured<JSObject>(src->zone(), dstKind);
size_t srcSize = Arena::thingSize(dstKind);
size_t tenuredSize = srcSize;
size_t dstSize = srcSize;
/*
* Arrays do not necessarily have the same AllocKind between src and dst.
@ -2958,7 +2945,7 @@ js::TenuringTracer::moveObjectToTenured(JSObject* dst, JSObject* src, AllocKind
* even if they are inlined.
*/
if (src->is<ArrayObject>()) {
tenuredSize = srcSize = sizeof(NativeObject);
dstSize = srcSize = sizeof(NativeObject);
} else if (src->is<TypedArrayObject>()) {
TypedArrayObject* tarray = &src->as<TypedArrayObject>();
// Typed arrays with inline data do not necessarily have the same
@ -2976,6 +2963,8 @@ js::TenuringTracer::moveObjectToTenured(JSObject* dst, JSObject* src, AllocKind
}
}
tenuredSize += dstSize;
// Copy the Cell contents.
MOZ_ASSERT(OffsetToChunkEnd(src) >= ptrdiff_t(srcSize));
js_memcpy(dst, src, srcSize);
@ -3000,7 +2989,44 @@ js::TenuringTracer::moveObjectToTenured(JSObject* dst, JSObject* src, AllocKind
CanNurseryAllocateFinalizedClass(src->getClass()));
}
return tenuredSize;
RelocationOverlay* overlay = RelocationOverlay::fromCell(src);
overlay->forwardTo(dst);
insertIntoFixupList(overlay);
TracePromoteToTenured(src, dst);
return dst;
}
inline JSObject*
js::TenuringTracer::movePlainObjectToTenured(PlainObject* src)
{
// Fast path version of moveToTenuredSlow() for specialized for PlainObject.
MOZ_ASSERT(IsInsideNursery(src));
MOZ_ASSERT(!src->zone()->usedByHelperThread());
AllocKind dstKind = src->allocKindForTenure();
auto dst = allocTenured<PlainObject>(src->zone(), dstKind);
size_t srcSize = Arena::thingSize(dstKind);
tenuredSize += srcSize;
// Copy the Cell contents.
MOZ_ASSERT(OffsetToChunkEnd(src) >= ptrdiff_t(srcSize));
js_memcpy(dst, src, srcSize);
// Move the slots and elements.
tenuredSize += moveSlotsToTenured(dst, src, dstKind);
tenuredSize += moveElementsToTenured(dst, src, dstKind);
MOZ_ASSERT(!dst->getClass()->extObjectMovedOp());
RelocationOverlay* overlay = RelocationOverlay::fromCell(src);
overlay->forwardTo(dst);
insertIntoFixupList(overlay);
TracePromoteToTenured(src, dst);
return dst;
}
size_t
@ -3081,6 +3107,23 @@ js::TenuringTracer::moveElementsToTenured(NativeObject* dst, NativeObject* src,
return nslots * sizeof(HeapSlot);
}
void
js::Nursery::collectToFixedPoint(TenuringTracer& mover, TenureCountCache& tenureCounts)
{
for (RelocationOverlay* p = mover.head; p; p = p->next()) {
JSObject* obj = static_cast<JSObject*>(p->forwardingAddress());
mover.traceObject(obj);
TenureCount& entry = tenureCounts.findEntry(obj->groupRaw());
if (entry.group == obj->groupRaw()) {
entry.count++;
} else if (!entry.group) {
entry.group = obj->groupRaw();
entry.count = 1;
}
}
}
/*** IsMarked / IsAboutToBeFinalized **************************************************************/

56
js/src/gc/Nursery-inl.h
View File

@ -29,6 +29,62 @@ js::Nursery::getForwardedPointer(JSObject** ref)
return true;
}
inline void
js::Nursery::maybeSetForwardingPointer(JSTracer* trc, void* oldData, void* newData, bool direct)
{
if (trc->isTenuringTracer())
setForwardingPointerWhileTenuring(oldData, newData, direct);
}
inline void
js::Nursery::setForwardingPointerWhileTenuring(void* oldData, void* newData, bool direct)
{
if (isInside(oldData))
setForwardingPointer(oldData, newData, direct);
}
inline void
js::Nursery::setSlotsForwardingPointer(HeapSlot* oldSlots, HeapSlot* newSlots, uint32_t nslots)
{
// Slot arrays always have enough space for a forwarding pointer, since the
// number of slots is never zero.
MOZ_ASSERT(nslots > 0);
setDirectForwardingPointer(oldSlots, newSlots);
}
inline void
js::Nursery::setElementsForwardingPointer(ObjectElements* oldHeader, ObjectElements* newHeader,
uint32_t capacity)
{
// Only use a direct forwarding pointer if there is enough space for one.
setForwardingPointer(oldHeader->elements(), newHeader->elements(),
capacity > 0);
}
inline void
js::Nursery::setForwardingPointer(void* oldData, void* newData, bool direct)
{
if (direct) {
setDirectForwardingPointer(oldData, newData);
return;
}
setIndirectForwardingPointer(oldData, newData);
}
inline void
js::Nursery::setDirectForwardingPointer(void* oldData, void* newData)
{
MOZ_ASSERT(isInside(oldData));
// Bug 1196210: If a zero-capacity header lands in the last 2 words of a
// jemalloc chunk abutting the start of a nursery chunk, the (invalid)
// newData pointer will appear to be "inside" the nursery.
MOZ_ASSERT(!isInside(newData) || (uintptr_t(newData) & js::gc::ChunkMask) == 0);
*reinterpret_cast<void**>(oldData) = newData;
}
namespace js {
// The allocation methods below will not run the garbage collector. If the

38
js/src/gc/Nursery.cpp
View File

@ -392,7 +392,7 @@ js::Nursery::freeBuffer(void* buffer)
}
void
Nursery::setForwardingPointer(void* oldData, void* newData, bool direct)
Nursery::setIndirectForwardingPointer(void* oldData, void* newData)
{
MOZ_ASSERT(isInside(oldData));
@ -401,37 +401,15 @@ Nursery::setForwardingPointer(void* oldData, void* newData, bool direct)
// newData pointer will appear to be "inside" the nursery.
MOZ_ASSERT(!isInside(newData) || (uintptr_t(newData) & ChunkMask) == 0);
if (direct) {
*reinterpret_cast<void**>(oldData) = newData;
} else {
AutoEnterOOMUnsafeRegion oomUnsafe;
if (!forwardedBuffers.initialized() && !forwardedBuffers.init())
oomUnsafe.crash("Nursery::setForwardingPointer");
AutoEnterOOMUnsafeRegion oomUnsafe;
if (!forwardedBuffers.initialized() && !forwardedBuffers.init())
oomUnsafe.crash("Nursery::setForwardingPointer");
#ifdef DEBUG
if (ForwardedBufferMap::Ptr p = forwardedBuffers.lookup(oldData))
MOZ_ASSERT(p->value() == newData);
if (ForwardedBufferMap::Ptr p = forwardedBuffers.lookup(oldData))
MOZ_ASSERT(p->value() == newData);
#endif
if (!forwardedBuffers.put(oldData, newData))
oomUnsafe.crash("Nursery::setForwardingPointer");
}
}
void
Nursery::setSlotsForwardingPointer(HeapSlot* oldSlots, HeapSlot* newSlots, uint32_t nslots)
{
// Slot arrays always have enough space for a forwarding pointer, since the
// number of slots is never zero.
MOZ_ASSERT(nslots > 0);
setForwardingPointer(oldSlots, newSlots, /* direct = */ true);
}
void
Nursery::setElementsForwardingPointer(ObjectElements* oldHeader, ObjectElements* newHeader,
uint32_t capacity)
{
// Only use a direct forwarding pointer if there is enough space for one.
setForwardingPointer(oldHeader->elements(), newHeader->elements(),
capacity > 0);
if (!forwardedBuffers.put(oldData, newData))
oomUnsafe.crash("Nursery::setForwardingPointer");
}
#ifdef DEBUG

33
js/src/gc/Nursery.h
View File

@ -55,6 +55,7 @@ struct Zone;
namespace js {
class ObjectElements;
class PlainObject;
class NativeObject;
class Nursery;
class HeapSlot;
@ -98,8 +99,6 @@ class TenuringTracer : public JSTracer
template <typename T> void traverse(T** thingp);
template <typename T> void traverse(T* thingp);
void insertIntoFixupList(gc::RelocationOverlay* entry);
// The store buffers need to be able to call these directly.
void traceObject(JSObject* src);
void traceObjectSlots(NativeObject* nobj, uint32_t start, uint32_t length);
@ -108,8 +107,12 @@ class TenuringTracer : public JSTracer
private:
Nursery& nursery() { return nursery_; }
JSObject* moveToTenured(JSObject* src);
size_t moveObjectToTenured(JSObject* dst, JSObject* src, gc::AllocKind dstKind);
inline void insertIntoFixupList(gc::RelocationOverlay* entry);
template <typename T>
inline T* allocTenured(JS::Zone* zone, gc::AllocKind kind);
inline JSObject* movePlainObjectToTenured(PlainObject* src);
JSObject* moveToTenuredSlow(JSObject* src);
size_t moveElementsToTenured(NativeObject* dst, NativeObject* src, gc::AllocKind dstKind);
size_t moveSlotsToTenured(NativeObject* dst, NativeObject* src, gc::AllocKind dstKind);
@ -216,15 +219,8 @@ class Nursery
/* Forward a slots/elements pointer stored in an Ion frame. */
void forwardBufferPointer(HeapSlot** pSlotsElems);
void maybeSetForwardingPointer(JSTracer* trc, void* oldData, void* newData, bool direct) {
if (trc->isTenuringTracer())
setForwardingPointerWhileTenuring(oldData, newData, direct);
}
void setForwardingPointerWhileTenuring(void* oldData, void* newData, bool direct) {
if (isInside(oldData))
setForwardingPointer(oldData, newData, direct);
}
inline void maybeSetForwardingPointer(JSTracer* trc, void* oldData, void* newData, bool direct);
inline void setForwardingPointerWhileTenuring(void* oldData, void* newData, bool direct);
/* Mark a malloced buffer as no longer needing to be freed. */
void removeMallocedBuffer(void* buffer) {
@ -475,11 +471,14 @@ class Nursery
void collectToFixedPoint(TenuringTracer& trc, gc::TenureCountCache& tenureCounts);
/* Handle relocation of slots/elements pointers stored in Ion frames. */
void setForwardingPointer(void* oldData, void* newData, bool direct);
inline void setForwardingPointer(void* oldData, void* newData, bool direct);
void setSlotsForwardingPointer(HeapSlot* oldSlots, HeapSlot* newSlots, uint32_t nslots);
void setElementsForwardingPointer(ObjectElements* oldHeader, ObjectElements* newHeader,
uint32_t capacity);
inline void setDirectForwardingPointer(void* oldData, void* newData);
void setIndirectForwardingPointer(void* oldData, void* newData);
inline void setSlotsForwardingPointer(HeapSlot* oldSlots, HeapSlot* newSlots, uint32_t nslots);
inline void setElementsForwardingPointer(ObjectElements* oldHeader, ObjectElements* newHeader,
uint32_t capacity);
/* Free malloced pointers owned by freed things in the nursery. */
void freeMallocedBuffers();

22
js/src/jsobj.cpp
View File

@ -3871,6 +3871,8 @@ js::DumpBacktrace(JSContext* cx)
js::gc::AllocKind
JSObject::allocKindForTenure(const js::Nursery& nursery) const
{
MOZ_ASSERT(IsInsideNursery(this));
if (is<ArrayObject>()) {
const ArrayObject& aobj = as<ArrayObject>();
MOZ_ASSERT(aobj.numFixedSlots() == 0);
@ -3883,6 +3885,12 @@ JSObject::allocKindForTenure(const js::Nursery& nursery) const
return GetBackgroundAllocKind(GetGCArrayKind(nelements));
}
// Unboxed plain objects are sized according to the data they store.
if (is<UnboxedPlainObject>()) {
size_t nbytes = as<UnboxedPlainObject>().layoutDontCheckGeneration().size();
return GetGCObjectKindForBytes(UnboxedPlainObject::offsetOfData() + nbytes);
}
if (is<JSFunction>())
return as<JSFunction>().getAllocKind();
@ -3901,12 +3909,6 @@ JSObject::allocKindForTenure(const js::Nursery& nursery) const
if (IsProxy(this))
return as<ProxyObject>().allocKindForTenure();
// Unboxed plain objects are sized according to the data they store.
if (is<UnboxedPlainObject>()) {
size_t nbytes = as<UnboxedPlainObject>().layoutDontCheckGeneration().size();
return GetGCObjectKindForBytes(UnboxedPlainObject::offsetOfData() + nbytes);
}
// Inlined typed objects are followed by their data, so make sure we copy
// it all over to the new object.
if (is<InlineTypedObject>()) {
@ -3923,13 +3925,7 @@ JSObject::allocKindForTenure(const js::Nursery& nursery) const
return AllocKind::OBJECT0;
// All nursery allocatable non-native objects are handled above.
MOZ_ASSERT(isNative());
AllocKind kind = GetGCObjectFixedSlotsKind(as<NativeObject>().numFixedSlots());
MOZ_ASSERT(!IsBackgroundFinalized(kind));
if (!CanBeFinalizedInBackground(kind, getClass()))
return kind;
return GetBackgroundAllocKind(kind);
return as<NativeObject>().allocKindForTenure();
}
void

1
js/src/vm/ArrayBufferObject.cpp
View File

@ -52,6 +52,7 @@
#include "jsatominlines.h"
#include "gc/Nursery-inl.h"
#include "vm/NativeObject-inl.h"
#include "vm/Shape-inl.h"

21
js/src/vm/NativeObject-inl.h
View File

@ -645,6 +645,27 @@ NativeObject::setLastProperty(JSContext* cx, Shape* shape)
return true;
}
inline js::gc::AllocKind
NativeObject::allocKindForTenure() const
{
using namespace js::gc;
AllocKind kind = GetGCObjectFixedSlotsKind(numFixedSlots());
MOZ_ASSERT(!IsBackgroundFinalized(kind));
if (!CanBeFinalizedInBackground(kind, getClass()))
return kind;
return GetBackgroundAllocKind(kind);
}
inline js::gc::AllocKind
PlainObject::allocKindForTenure() const
{
using namespace js::gc;
AllocKind kind = GetGCObjectFixedSlotsKind(numFixedSlots());
MOZ_ASSERT(!IsBackgroundFinalized(kind));
MOZ_ASSERT(CanBeFinalizedInBackground(kind, getClass()));
return GetBackgroundAllocKind(kind);
}
/* Make an object with pregenerated shape from a NEWOBJECT bytecode. */
static inline PlainObject*
CopyInitializerObject(JSContext* cx, HandlePlainObject baseobj, NewObjectKind newKind = GenericObject)

6
js/src/vm/NativeObject.h
View File

@ -1400,6 +1400,9 @@ class NativeObject : public ShapedObject
copy(JSContext* cx, gc::AllocKind kind, gc::InitialHeap heap,
HandleNativeObject templateObject);
/* Return the allocKind we would use if we were to tenure this object. */
inline js::gc::AllocKind allocKindForTenure() const;
void updateShapeAfterMovingGC();
void sweepDictionaryListPointer();
void updateDictionaryListPointerAfterMinorGC(NativeObject* old);
@ -1423,6 +1426,9 @@ class PlainObject : public NativeObject
{
public:
static const js::Class class_;
/* Return the allocKind we would use if we were to tenure this object. */
inline js::gc::AllocKind allocKindForTenure() const;
};
inline void