Bug 1167452 - Barrier weakmap operations and maintain weak keys table during incremental collections. r=jonco

Differential Revision: https://phabricator.services.mozilla.com/D31958

--HG--
extra : moz-landing-system : lando
This commit is contained in:
Steve Fink 2019-05-31 23:33:48 +00:00
parent bb748896e1
commit 6eee7b5e6f
17 changed files with 554 additions and 200 deletions

View File

@ -66,12 +66,6 @@ class JS_PUBLIC_API JSTracer {
// everything reachable by regular edges has been marked.
Marking,
// Same as Marking, except we have now moved on to the "weak marking
// phase", in which every marked obj/script is immediately looked up to
// see if it is a weak map key (and therefore might require marking its
// weak map value).
WeakMarking,
// A tracer that traverses the graph for the purposes of moving objects
// from the nursery to the tenured area.
Tenuring,
@ -80,12 +74,7 @@ class JS_PUBLIC_API JSTracer {
// Traversing children is the responsibility of the callback.
Callback
};
bool isMarkingTracer() const {
return tag_ == TracerKindTag::Marking || tag_ == TracerKindTag::WeakMarking;
}
bool isWeakMarkingTracer() const {
return tag_ == TracerKindTag::WeakMarking;
}
bool isMarkingTracer() const { return tag_ == TracerKindTag::Marking; }
bool isTenuringTracer() const { return tag_ == TracerKindTag::Tenuring; }
bool isCallbackTracer() const { return tag_ == TracerKindTag::Callback; }
inline JS::CallbackTracer* asCallbackTracer();

View File

@ -439,6 +439,34 @@ static MOZ_ALWAYS_INLINE void AssertValidToSkipBarrier(TenuredCell* thing) {
AssertValidToSkipBarrier(next);
}
// Like gc::MarkColor but allows the possibility of the cell being
// unmarked. Order is important here, with white being 'least marked'
// and black being 'most marked'.
enum class CellColor : uint8_t { White = 0, Gray = 1, Black = 2 };
inline CellColor GetCellColor(Cell* cell) {
if (cell->isMarkedBlack()) {
return CellColor::Black;
}
if (cell->isMarkedGray()) {
return CellColor::Gray;
}
return CellColor::White;
}
static inline CellColor GetCellColor(MarkColor color) {
return color == MarkColor::Black ? CellColor::Black : CellColor::Gray;
}
static inline MarkColor GetMarkColor(CellColor color) {
MOZ_ASSERT(color != CellColor::White);
return color == CellColor::Black ? MarkColor::Black : MarkColor::Gray;
}
static inline bool IsMarked(CellColor c) { return c != CellColor::White; }
#ifdef DEBUG
/* static */ void Cell::assertThingIsNotGray(Cell* cell) {

View File

@ -4604,7 +4604,7 @@ void GCRuntime::markWeakReferences(gcstats::PhaseKind phase) {
for (;;) {
bool markedAny = false;
if (!marker.isWeakMarkingTracer()) {
if (!marker.isWeakMarking()) {
for (ZoneIterT zone(rt); !zone.done(); zone.next()) {
markedAny |= WeakMapBase::markZoneIteratively(zone, &marker);
}
@ -4733,7 +4733,7 @@ void js::gc::MarkingValidator::nonIncrementalMark(AutoGCSession& session) {
* collecting.
*/
WeakMapSet markedWeakMaps;
WeakMapColors markedWeakMaps;
/*
* For saving, smush all of the keys into one big table and split them back

View File

@ -225,6 +225,26 @@ class MarkStackIter {
} /* namespace gc */
enum MarkingState : uint8_t {
// Have not yet started marking.
NotActive,
// Main marking mode. Weakmap marking will be populating the weakKeys tables
// but not consulting them. The state will transition to WeakMarking until it
// is done, then back to RegularMarking.
RegularMarking,
// Same as RegularMarking except now every marked obj/script is immediately
// looked up in the weakKeys table to see if it is a weakmap key, and
// therefore might require marking its value. Transitions back to
// RegularMarking when done.
WeakMarking,
// Same as RegularMarking, but we OOMed (or obeyed a directive in the test
// marking queue) and fell back to iterating until the next GC.
IterativeMarking
};
class GCMarker : public JSTracer {
public:
explicit GCMarker(JSRuntime* rt);
@ -292,11 +312,21 @@ class GCMarker : public JSTracer {
void leaveWeakMarkingMode();
void abortLinearWeakMarking() {
leaveWeakMarkingMode();
linearWeakMarkingDisabled_ = true;
state = MarkingState::IterativeMarking;
}
void delayMarkingChildren(gc::Cell* cell);
// Remove <map,toRemove> from the weak keys table indexed by 'key'.
void forgetWeakKey(js::gc::WeakKeyTable& weakKeys, WeakMapBase* map,
gc::Cell* keyOrDelegate, gc::Cell* keyToRemove);
// Purge all mention of 'map' from the weak keys table.
void forgetWeakMap(WeakMapBase* map, Zone* zone);
// 'delegate' is no longer the delegate of 'key'.
void severWeakDelegate(JSObject* key, JSObject* delegate);
bool isDrained() { return isMarkStackEmpty() && !delayedMarkingList; }
// The mark queue is a testing-only feature for controlling mark ordering and
@ -334,6 +364,8 @@ class GCMarker : public JSTracer {
template <typename T>
void markImplicitEdges(T* oldThing);
bool isWeakMarking() const { return state == MarkingState::WeakMarking; }
private:
#ifdef DEBUG
void checkZone(void* p);
@ -423,22 +455,16 @@ class GCMarker : public JSTracer {
/* Whether more work has been added to the delayed marking list. */
MainThreadData<bool> delayedMarkingWorkAdded;
/*
* If the weakKeys table OOMs, disable the linear algorithm and fall back
* to iterating until the next GC.
*/
MainThreadData<bool> linearWeakMarkingDisabled_;
/* The count of marked objects during GC. */
size_t markCount;
/* Track the state of marking. */
MainThreadData<MarkingState> state;
#ifdef DEBUG
/* Count of arenas that are currently in the stack. */
MainThreadData<size_t> markLaterArenas;
/* Assert that start and stop are called with correct ordering. */
MainThreadData<bool> started;
/* The test marking queue might want to be marking a particular color. */
mozilla::Maybe<js::gc::MarkColor> queueMarkColor;
@ -485,6 +511,11 @@ class MOZ_RAII AutoSetMarkColor {
marker_.setMarkColor(newColor);
}
AutoSetMarkColor(GCMarker& marker, CellColor newColor)
: AutoSetMarkColor(marker, GetMarkColor(newColor)) {
MOZ_ASSERT(newColor != CellColor::White);
}
~AutoSetMarkColor() { marker_.setMarkColor(initialColor_); }
};

View File

@ -110,8 +110,12 @@ PhaseKindGraphRoots = [
UnmarkGrayPhaseKind,
]),
PhaseKind("SWEEP_MARK_INCOMING_GRAY", "Mark Incoming Gray Pointers", 14),
PhaseKind("SWEEP_MARK_GRAY", "Mark Gray", 15),
PhaseKind("SWEEP_MARK_GRAY_WEAK", "Mark Gray and Weak", 16)
PhaseKind("SWEEP_MARK_GRAY", "Mark Gray", 15, [
UnmarkGrayPhaseKind,
]),
PhaseKind("SWEEP_MARK_GRAY_WEAK", "Mark Gray and Weak", 16, [
UnmarkGrayPhaseKind,
]),
]),
PhaseKind("FINALIZE_START", "Finalize Start Callbacks", 17, [
PhaseKind("WEAK_ZONES_CALLBACK", "Per-Slice Weak Callback", 57),

View File

@ -35,6 +35,7 @@
#include "gc/GC-inl.h"
#include "gc/Nursery-inl.h"
#include "gc/PrivateIterators-inl.h"
#include "gc/WeakMap-inl.h"
#include "gc/Zone-inl.h"
#include "vm/GeckoProfiler-inl.h"
#include "vm/NativeObject-inl.h"
@ -609,11 +610,6 @@ void GCMarker::markEphemeronValues(gc::Cell* markedCell,
DebugOnly<size_t> initialLen = values.length();
for (const auto& markable : values) {
if (color == gc::MarkColor::Black &&
markable.weakmap->markColor == gc::MarkColor::Gray) {
continue;
}
markable.weakmap->markEntry(this, markedCell, markable.key);
}
@ -623,9 +619,62 @@ void GCMarker::markEphemeronValues(gc::Cell* markedCell,
MOZ_ASSERT(values.length() == initialLen);
}
void GCMarker::forgetWeakKey(js::gc::WeakKeyTable& weakKeys, WeakMapBase* map,
gc::Cell* keyOrDelegate, gc::Cell* keyToRemove) {
// Find and remove the exact pair <map,keyToRemove> from the values of the
// weak keys table.
//
// This function is called when 'keyToRemove' is removed from a weakmap
// 'map'. If 'keyToRemove' has a delegate, then the delegate will be used as
// the lookup key in gcWeakKeys; otherwise, 'keyToRemove' itself will be. In
// either case, 'keyToRemove' is what we will be filtering out of the
// Markable values in the weakKey table.
auto p = weakKeys.get(keyOrDelegate);
// Note that this is not guaranteed to find anything. The key will have
// only been inserted into the weakKeys table if it was unmarked when the
// map was traced.
if (p) {
EraseIf(p->value, [map, keyToRemove](const WeakMarkable& markable) -> bool {
// Note that we should only have had the key in weakKeys if the map
// was marked.
MOZ_ASSERT(IsMarked(markable.weakmap->markColor));
return (markable.weakmap == map) && (markable.key == keyToRemove);
});
}
} // namespace js
void GCMarker::forgetWeakMap(WeakMapBase* map, Zone* zone) {
for (auto p = zone->gcNurseryWeakKeys().all(); !p.empty(); p.popFront()) {
EraseIf(p.front().value, [map](const WeakMarkable& markable) -> bool {
return markable.weakmap == map;
});
}
for (auto p = zone->gcWeakKeys().all(); !p.empty(); p.popFront()) {
EraseIf(p.front().value, [map](const WeakMarkable& markable) -> bool {
return markable.weakmap == map;
});
}
}
// 'delegate' is no longer the delegate of 'key'.
void GCMarker::severWeakDelegate(JSObject* key, JSObject* delegate) {
JS::Zone* zone = delegate->zone();
auto p = zone->gcWeakKeys(delegate).get(delegate);
if (p) {
EraseIf(p->value, [this, key](const WeakMarkable& markable) -> bool {
if (markable.key != key) {
return false;
}
markable.weakmap->postSeverDelegate(this, key, key->compartment());
return true;
});
}
}
template <typename T>
void GCMarker::markImplicitEdgesHelper(T markedThing) {
if (!isWeakMarkingTracer()) {
if (state != MarkingState::WeakMarking) {
return;
}
@ -1527,7 +1576,7 @@ GCMarker::MarkQueueProgress GCMarker::processMarkQueue() {
return QueueYielded;
} else if (js::StringEqualsAscii(str, "enter-weak-marking-mode") ||
js::StringEqualsAscii(str, "abort-weak-marking-mode")) {
if (!isWeakMarkingTracer() && !linearWeakMarkingDisabled_) {
if (state == MarkingState::RegularMarking) {
// We can't enter weak marking mode at just any time, so instead
// we'll stop processing the queue and continue on with the GC. Once
// we enter weak marking mode, we can continue to the rest of the
@ -2359,11 +2408,11 @@ GCMarker::GCMarker(JSRuntime* rt)
grayStack(),
color(MarkColor::Black),
delayedMarkingList(nullptr),
delayedMarkingWorkAdded(false)
delayedMarkingWorkAdded(false),
state(MarkingState::NotActive)
#ifdef DEBUG
,
markLaterArenas(0),
started(false),
strictCompartmentChecking(false),
markQueue(rt),
queuePos(0)
@ -2376,12 +2425,9 @@ bool GCMarker::init(JSGCMode gcMode) {
}
void GCMarker::start() {
#ifdef DEBUG
MOZ_ASSERT(!started);
started = true;
#endif
MOZ_ASSERT(state == MarkingState::NotActive);
state = MarkingState::RegularMarking;
color = MarkColor::Black;
linearWeakMarkingDisabled_ = false;
#ifdef DEBUG
queuePos = 0;
@ -2393,15 +2439,11 @@ void GCMarker::start() {
}
void GCMarker::stop() {
#ifdef DEBUG
MOZ_ASSERT(isDrained());
MOZ_ASSERT(started);
started = false;
MOZ_ASSERT(!delayedMarkingList);
MOZ_ASSERT(markLaterArenas == 0);
#endif
MOZ_ASSERT(state != MarkingState::NotActive);
state = MarkingState::NotActive;
/* Free non-ballast stack memory. */
blackStack.clear();
@ -2502,50 +2544,80 @@ void GCMarker::repush(JSObject* obj) {
void GCMarker::enterWeakMarkingMode() {
MOZ_ASSERT(runtime()->gc.nursery().isEmpty());
MOZ_ASSERT(tag_ == TracerKindTag::Marking);
if (linearWeakMarkingDisabled_) {
MOZ_ASSERT(isMarkingTracer());
if (state != MarkingState::RegularMarking) {
return;
}
// During weak marking mode, we maintain a table mapping weak keys to
// entries in known-live weakmaps. Initialize it with the keys of marked
// weakmaps -- or more precisely, the keys of marked weakmaps that are
// mapped to not yet live values. (Once bug 1167452 implements incremental
// weakmap marking, this initialization step will become unnecessary, as
// the table will already hold all such keys.)
if (weakMapAction() == ExpandWeakMaps) {
tag_ = TracerKindTag::WeakMarking;
if (weakMapAction() != ExpandWeakMaps) {
return; // This marker does not do linear-time weak marking.
}
// If there was an 'enter-weak-marking-mode' token in the queue, then it
// and everything after it will still be in the queue so we can process
// them now.
while (processMarkQueue() == QueueYielded) {
};
// Set state before doing anything else, so any new key that is marked
// during the following gcWeakKeys scan will itself be looked up in
// gcWeakKeys and marked according to ephemeron rules.
state = MarkingState::WeakMarking;
for (SweepGroupZonesIter zone(runtime()); !zone.done(); zone.next()) {
for (WeakMapBase* m : zone->gcWeakMapList()) {
if (m->marked) {
(void)m->markEntries(this);
// If there was an 'enter-weak-marking-mode' token in the queue, then it
// and everything after it will still be in the queue so we can process
// them now.
while (processMarkQueue() == QueueYielded) {
};
// gcWeakKeys contains the keys from all weakmaps marked so far, or at least
// the keys that might still need to be marked through. Scan through
// gcWeakKeys and mark all values whose keys are marked. This marking may
// recursively mark through other weakmap entries (immediately since we are
// now in WeakMarking mode). The end result is a consistent state where all
// values are marked if both their map and key are marked -- though note that
// we may later leave weak marking mode, do some more marking, and then enter
// back in.
for (SweepGroupZonesIter zone(runtime(), js::SkipAtoms); !zone.done();
zone.next()) {
if (!zone->isGCMarking()) {
continue;
}
MOZ_ASSERT(zone->gcNurseryWeakKeys().count() == 0);
// An OrderedHashMap::Range stays valid even when the underlying table
// (zone->gcWeakKeys) is mutated, which is useful here since we may add
// additional entries while iterating over the Range.
gc::WeakKeyTable::Range r = zone->gcWeakKeys().all();
while (!r.empty()) {
gc::Cell* key = r.front().key;
if (key->isMarkedAny()) {
MOZ_ASSERT(key == r.front().key);
auto& markables = r.front().value;
r.popFront(); // Pop before any mutations happen.
size_t end = markables.length();
for (size_t i = 0; i < end; i++) {
WeakMarkable& v = markables[i];
// Note: if the key is marked gray but not black, then the markables
// vector may be appended to within this loop body. So iterate just
// over the ones from before weak marking mode was switched on.
v.weakmap->markEntry(this, key, v.key);
}
markables.erase(markables.begin(), end < markables.length()
? &markables[end]
: markables.end());
} else {
r.popFront();
}
}
}
}
void GCMarker::leaveWeakMarkingMode() {
MOZ_ASSERT_IF(
weakMapAction() == ExpandWeakMaps && !linearWeakMarkingDisabled_,
tag_ == TracerKindTag::WeakMarking);
tag_ = TracerKindTag::Marking;
// Table is expensive to maintain when not in weak marking mode, so we'll
// rebuild it upon entry rather than allow it to contain stale data.
AutoEnterOOMUnsafeRegion oomUnsafe;
for (GCZonesIter zone(runtime()); !zone.done(); zone.next()) {
if (!zone->gcWeakKeys().clear()) {
oomUnsafe.crash("clearing weak keys in GCMarker::leaveWeakMarkingMode()");
}
MOZ_ASSERT_IF(weakMapAction() == ExpandWeakMaps,
state == MarkingState::WeakMarking ||
state == MarkingState::IterativeMarking);
if (state != MarkingState::IterativeMarking) {
state = MarkingState::RegularMarking;
}
// The gcWeakKeys table is still populated and may be used during a future
// weak marking mode within this GC.
}
void GCMarker::delayMarkingChildren(Cell* cell) {
@ -2694,7 +2766,7 @@ void gc::PushArena(GCMarker* gcmarker, Arena* arena) {
#ifdef DEBUG
void GCMarker::checkZone(void* p) {
MOZ_ASSERT(started);
MOZ_ASSERT(state != MarkingState::NotActive);
DebugOnly<Cell*> cell = static_cast<Cell*>(p);
MOZ_ASSERT_IF(cell->isTenured(), cell->asTenured().zone()->isCollecting());
}

View File

@ -763,8 +763,7 @@ bool js::gc::CheckWeakMapEntryMarking(const WeakMapBase* map, Cell* key,
Zone* valueZone = GetCellZoneFromAnyThread(value);
MOZ_ASSERT(valueZone == zone || valueZone->isAtomsZone());
CellColor mapColor =
map->markColor == MarkColor::Black ? CellColor::Black : CellColor::Gray;
CellColor mapColor = map->markColor;
if (object && GetCellColor(object) != mapColor) {
fprintf(stderr, "WeakMap object is marked differently to the map\n");
fprintf(stderr, "(map %p is %s, object %p is %s)\n", map,

View File

@ -16,32 +16,12 @@
namespace js {
namespace gc {
// Like gc::MarkColor but allows the possibility of the cell being
// unmarked.
enum class CellColor : uint8_t {
White = 0,
Gray = uint8_t(MarkColor::Gray),
Black = uint8_t(MarkColor::Black)
};
static constexpr CellColor AllCellColors[] = {CellColor::White, CellColor::Gray,
CellColor::Black};
static constexpr CellColor MarkedCellColors[] = {CellColor::Gray,
CellColor::Black};
inline CellColor GetCellColor(Cell* cell) {
if (cell->isMarkedBlack()) {
return CellColor::Black;
}
if (cell->isMarkedGray()) {
return CellColor::Gray;
}
return CellColor::White;
}
inline CellColor ExpectedWeakMapValueColor(CellColor keyColor,
CellColor mapColor) {
return Min(keyColor, mapColor);

View File

@ -8,12 +8,15 @@
#define gc_WeakMap_inl_h
#include "gc/WeakMap.h"
#include "gc/PublicIterators.h"
#include "gc/Zone.h"
#include "js/TraceKind.h"
#include "vm/JSContext.h"
namespace js {
namespace gc {
namespace detail {
template <typename T>
static T extractUnbarriered(const WriteBarriered<T>& v) {
@ -25,18 +28,45 @@ static T* extractUnbarriered(T* v) {
return v;
}
inline /* static */ JSObject* WeakMapBase::getDelegate(JSObject* key) {
return UncheckedUnwrapWithoutExpose(key);
template <typename T>
static JS::Zone* GetZone(T t) {
return t->zone();
}
inline /* static */ JSObject* WeakMapBase::getDelegate(JSScript* script) {
static JS::Zone* GetZone(js::HeapPtr<JS::Value>& t) {
if (!t.isGCThing()) {
return nullptr;
}
return t.toGCThing()->asTenured().zone();
}
// Only objects have delegates, so default to returning nullptr. Note that some
// compilation units will only ever use the object version.
static MOZ_MAYBE_UNUSED JSObject* GetDelegateHelper(gc::Cell* key) {
return nullptr;
}
inline /* static */ JSObject* WeakMapBase::getDelegate(LazyScript* script) {
return nullptr;
static JSObject* GetDelegateHelper(JSObject* key) {
JSObject* delegate = UncheckedUnwrapWithoutExpose(key);
return (key == delegate) ? nullptr : delegate;
}
} /* namespace detail */
} /* namespace gc */
// Use a helper function to do overload resolution to handle cases like
// Heap<ObjectSubclass*>: find everything that is convertible to JSObject* (and
// avoid calling barriers).
template <typename T>
inline /* static */ JSObject* WeakMapBase::getDelegate(const T& key) {
using namespace gc::detail;
return GetDelegateHelper(extractUnbarriered(key));
}
template <>
inline /* static */ JSObject* WeakMapBase::getDelegate(gc::Cell* const&) =
delete;
template <class K, class V>
WeakMap<K, V>::WeakMap(JSContext* cx, JSObject* memOf)
: Base(cx->zone()), WeakMapBase(memOf, cx->zone()) {
@ -52,11 +82,22 @@ WeakMap<K, V>::WeakMap(JSContext* cx, JSObject* memOf)
zone()->gcWeakMapList().insertFront(this);
if (zone()->wasGCStarted()) {
marked = true;
markColor = gc::MarkColor::Black;
markColor = CellColor::Black;
}
}
namespace gc {
// Compute the correct color to mark a weakmap entry value based on the map and
// key colors.
struct AutoSetValueColor : gc::AutoSetMarkColor {
AutoSetValueColor(GCMarker& marker, CellColor mapColor, CellColor keyColor)
: gc::AutoSetMarkColor(
marker, mapColor == keyColor ? mapColor : CellColor::Gray) {}
};
} // namespace gc
// Trace a WeakMap entry based on 'markedCell' getting marked, where 'origKey'
// is the key in the weakmap. These will probably be the same, but can be
// different eg when markedCell is a delegate for origKey.
@ -64,25 +105,40 @@ WeakMap<K, V>::WeakMap(JSContext* cx, JSObject* memOf)
// This implementation does not use 'markedCell'; it looks up origKey and checks
// the mark bits on everything it cares about, one of which will be
// markedCell. But a subclass might use it to optimize the liveness check.
//
// markEntry is called when encountering a weakmap key during marking, or when
// entering weak marking mode.
template <class K, class V>
void WeakMap<K, V>::markEntry(GCMarker* marker, gc::Cell* markedCell,
gc::Cell* origKey) {
MOZ_ASSERT(marked);
using namespace gc::detail;
Ptr p = Base::lookup(static_cast<Lookup>(origKey));
// We should only be processing <weakmap,key> pairs where the key exists in
// the weakmap. Such pairs are inserted when a weakmap is marked, and are
// removed by barriers if the key is removed from the weakmap. Failure here
// probably means gcWeakKeys is not being properly traced during a minor GC,
// or the weakmap keys are not being updated when tenured.
MOZ_ASSERT(p.found());
K key(p->key());
MOZ_ASSERT((markedCell == extractUnbarriered(key)) ||
(markedCell == getDelegate(key)));
if (marker->isMarked(&key)) {
TraceEdge(marker, &p->value(), "ephemeron value");
} else if (keyNeedsMark(marker, key)) {
TraceEdge(marker, &p->value(), "WeakMap ephemeron value");
CellColor delegateColor = getDelegateColor(key);
if (IsMarked(delegateColor) && key->zone()->isGCMarking()) {
gc::AutoSetMarkColor autoColor(*marker, delegateColor);
TraceEdge(marker, &key, "proxy-preserved WeakMap ephemeron key");
MOZ_ASSERT(key == p->key()); // No moving
MOZ_ASSERT(key == p->key(), "no moving GC");
}
CellColor keyColor = getCellColor(key);
if (IsMarked(keyColor)) {
JS::Zone* valueZone = GetZone(p->value());
if (valueZone && valueZone->isGCMarking()) {
gc::AutoSetValueColor autoColor(*marker, markColor, keyColor);
TraceEdge(marker, &p->value(), "WeakMap ephemeron value");
}
}
key.unsafeSet(nullptr); // Prevent destructor from running barriers.
}
template <class K, class V>
@ -98,13 +154,12 @@ void WeakMap<K, V>::trace(JSTracer* trc) {
// Don't change the map color from black to gray. This can happen when a
// barrier pushes the map object onto the black mark stack when it's already
// present on the gray mark stack, which is marked later.
if (marked && markColor == gc::MarkColor::Black &&
if (markColor == CellColor::Black &&
marker->markColor() == gc::MarkColor::Gray) {
return;
}
marked = true;
markColor = marker->markColor();
markColor = GetCellColor(marker->markColor());
(void)markEntries(marker);
return;
}
@ -131,8 +186,7 @@ template <class K, class V>
/* static */ void WeakMap<K, V>::addWeakEntry(
GCMarker* marker, gc::Cell* key, const gc::WeakMarkable& markable) {
Zone* zone = key->asTenured().zone();
auto& weakKeys =
gc::IsInsideNursery(key) ? zone->gcNurseryWeakKeys() : zone->gcWeakKeys();
auto& weakKeys = zone->gcWeakKeys(key);
auto p = weakKeys.get(key);
if (p) {
gc::WeakEntryVector& weakEntries = p->value;
@ -150,39 +204,45 @@ template <class K, class V>
template <class K, class V>
bool WeakMap<K, V>::markEntries(GCMarker* marker) {
MOZ_ASSERT(marked);
if (marker->markColor() == gc::MarkColor::Black &&
markColor == gc::MarkColor::Gray) {
return false;
}
MOZ_ASSERT(IsMarked(markColor));
bool markedAny = false;
for (Enum e(*this); !e.empty(); e.popFront()) {
// If the entry is live, ensure its key and value are marked.
bool keyIsMarked = marker->isMarked(&e.front().mutableKey());
if (!keyIsMarked && keyNeedsMark(marker, e.front().key())) {
CellColor keyColor = getCellColor(e.front().key().get());
CellColor delegateColor = getDelegateColor(e.front().key());
if (IsMarked(delegateColor) && keyColor < delegateColor) {
gc::AutoSetMarkColor autoColor(*marker, delegateColor);
TraceEdge(marker, &e.front().mutableKey(),
"proxy-preserved WeakMap entry key");
keyIsMarked = true;
markedAny = true;
keyColor = delegateColor;
}
if (keyIsMarked) {
if (IsMarked(keyColor)) {
gc::AutoSetValueColor autoColor(*marker, markColor, keyColor);
if (!marker->isMarked(&e.front().value())) {
TraceEdge(marker, &e.front().value(), "WeakMap entry value");
markedAny = true;
}
} else if (marker->isWeakMarkingTracer()) {
// Entry is not yet known to be live. Record this weakmap and
// the lookup key in the list of weak keys. Also record the
// delegate, if any, because marking the delegate also marks
// the entry.
gc::Cell* weakKey = extractUnbarriered(e.front().key());
}
// Changes in the map's mark color will be handled in this code, but
// changes in the key's mark color are handled through the weak keys table.
// So we only need to populate the table if the key is less marked than the
// map, to catch later updates in the key's mark color.
if (keyColor < markColor) {
MOZ_ASSERT(marker->weakMapAction() == ExpandWeakMaps);
// Entry is not yet known to be live. Record this weakmap and the lookup
// key in the list of weak keys. If the key has a delegate, then the
// lookup key is the delegate (because marking the key will end up
// marking the delegate and thereby mark the entry.)
gc::Cell* weakKey = gc::detail::extractUnbarriered(e.front().key());
gc::WeakMarkable markable(this, weakKey);
addWeakEntry(marker, weakKey, markable);
if (JSObject* delegate = getDelegate(e.front().key())) {
addWeakEntry(marker, delegate, markable);
} else {
addWeakEntry(marker, weakKey, markable);
}
}
}
@ -191,25 +251,33 @@ bool WeakMap<K, V>::markEntries(GCMarker* marker) {
}
template <class K, class V>
inline bool WeakMap<K, V>::keyNeedsMark(GCMarker* marker, JSObject* key) const {
void WeakMap<K, V>::postSeverDelegate(GCMarker* marker, gc::Cell* key,
Compartment* comp) {
if (IsMarked(markColor)) {
// We only stored the delegate, not the key, and we're severing the
// delegate from the key. So store the key.
gc::WeakMarkable markable(this, key);
addWeakEntry(marker, key, markable);
}
}
template <class K, class V>
inline WeakMapBase::CellColor WeakMap<K, V>::getDelegateColor(
JSObject* key) const {
JSObject* delegate = getDelegate(key);
/*
* Check if the delegate is marked with any color to properly handle
* gray marking when the key's delegate is black and the map is gray.
*/
return delegate && marker->isMarkedUnbarriered(&delegate);
return delegate ? getCellColor(delegate) : CellColor::White;
}
template <class K, class V>
inline bool WeakMap<K, V>::keyNeedsMark(GCMarker* marker,
JSScript* script) const {
return false;
inline WeakMapBase::CellColor WeakMap<K, V>::getDelegateColor(
JSScript* script) const {
return CellColor::White;
}
template <class K, class V>
inline bool WeakMap<K, V>::keyNeedsMark(GCMarker* marker,
LazyScript* script) const {
return false;
inline WeakMapBase::CellColor WeakMap<K, V>::getDelegateColor(
LazyScript* script) const {
return CellColor::White;
}
template <class K, class V>

View File

@ -23,7 +23,7 @@ using namespace js;
using namespace js::gc;
WeakMapBase::WeakMapBase(JSObject* memOf, Zone* zone)
: memberOf(memOf), zone_(zone), marked(false), markColor(MarkColor::Black) {
: memberOf(memOf), zone_(zone), markColor(CellColor::White) {
MOZ_ASSERT_IF(memberOf, memberOf->compartment()->zone() == zone);
}
@ -32,8 +32,14 @@ WeakMapBase::~WeakMapBase() {
}
void WeakMapBase::unmarkZone(JS::Zone* zone) {
AutoEnterOOMUnsafeRegion oomUnsafe;
if (!zone->gcWeakKeys().clear()) {
oomUnsafe.crash("clearing weak keys table");
}
MOZ_ASSERT(zone->gcNurseryWeakKeys().count() == 0);
for (WeakMapBase* m : zone->gcWeakMapList()) {
m->marked = false;
m->markColor = CellColor::White;
}
}
@ -52,7 +58,7 @@ bool WeakMapBase::checkMarkingForZone(JS::Zone* zone) {
bool ok = true;
for (WeakMapBase* m : zone->gcWeakMapList()) {
if (m->marked && !m->checkMarking()) {
if (IsMarked(m->markColor) && !m->checkMarking()) {
ok = false;
}
}
@ -64,7 +70,7 @@ bool WeakMapBase::checkMarkingForZone(JS::Zone* zone) {
bool WeakMapBase::markZoneIteratively(JS::Zone* zone, GCMarker* marker) {
bool markedAny = false;
for (WeakMapBase* m : zone->gcWeakMapList()) {
if (m->marked && m->markEntries(marker)) {
if (IsMarked(m->markColor) && m->markEntries(marker)) {
markedAny = true;
}
}
@ -83,7 +89,7 @@ bool WeakMapBase::findSweepGroupEdges(JS::Zone* zone) {
void WeakMapBase::sweepZone(JS::Zone* zone) {
for (WeakMapBase* m = zone->gcWeakMapList().getFirst(); m;) {
WeakMapBase* next = m->getNext();
if (m->marked) {
if (IsMarked(m->markColor)) {
m->sweep();
} else {
m->clearAndCompact();
@ -94,7 +100,7 @@ void WeakMapBase::sweepZone(JS::Zone* zone) {
#ifdef DEBUG
for (WeakMapBase* m : zone->gcWeakMapList()) {
MOZ_ASSERT(m->isInList() && m->marked);
MOZ_ASSERT(m->isInList() && IsMarked(m->markColor));
}
#endif
}
@ -111,21 +117,24 @@ void WeakMapBase::traceAllMappings(WeakMapTracer* tracer) {
}
bool WeakMapBase::saveZoneMarkedWeakMaps(JS::Zone* zone,
WeakMapSet& markedWeakMaps) {
WeakMapColors& markedWeakMaps) {
for (WeakMapBase* m : zone->gcWeakMapList()) {
if (m->marked && !markedWeakMaps.put(m)) {
return false;
if (IsMarked(m->markColor)) {
if (!markedWeakMaps.put(m, m->markColor)) {
return false;
}
}
}
return true;
}
void WeakMapBase::restoreMarkedWeakMaps(WeakMapSet& markedWeakMaps) {
for (WeakMapSet::Range r = markedWeakMaps.all(); !r.empty(); r.popFront()) {
WeakMapBase* map = r.front();
void WeakMapBase::restoreMarkedWeakMaps(WeakMapColors& markedWeakMaps) {
for (WeakMapColors::Range r = markedWeakMaps.all(); !r.empty();
r.popFront()) {
WeakMapBase* map = r.front().key();
MOZ_ASSERT(map->zone()->isGCMarking());
MOZ_ASSERT(!map->marked);
map->marked = true;
MOZ_ASSERT(map->markColor == CellColor::White);
map->markColor = r.front().value();
}
}
@ -134,17 +143,15 @@ size_t ObjectValueMap::sizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) {
}
bool ObjectValueMap::findZoneEdges() {
/*
* For unmarked weakmap keys with delegates in a different zone, add a zone
* edge to ensure that the delegate zone finishes marking before the key
* zone.
*/
// For weakmap keys with delegates in a different zone, add a zone edge to
// ensure that the delegate zone finishes marking before the key zone.
//
// Possibly add an edge the other direction too, because scanning the
// gcWeakKeys for a zone containing a delegate might end up marking a value
// in the map/key zone.
JS::AutoSuppressGCAnalysis nogc;
for (Range r = all(); !r.empty(); r.popFront()) {
JSObject* key = r.front().key();
if (key->asTenured().isMarkedBlack()) {
continue;
}
JSObject* delegate = getDelegate(key);
if (!delegate) {
continue;
@ -156,6 +163,30 @@ bool ObjectValueMap::findZoneEdges() {
if (!delegateZone->addSweepGroupEdgeTo(key->zone())) {
return false;
}
// The various cases depend on the order that the map and key are marked:
//
// If the key is marked:
// key marked, then map marked:
// - value was marked with map
// map marked, key already in map, key marked before weak marking mode:
// - key added to weakKeys when map marked
// - value marked during enterWeakMarkingMode: this requires the
// delegate's zone to be marked before the key zone, since only the
// delegate will be in weakKeys.
// map marked, key already in map, key marked after weak marking mode:
// - during key marking, weakKeys[key] triggers marking of value
// map marked, key inserted into map, key marked:
// - value marked by insert barrier
//
// In all cases, a marked key will have already marked the value, so there
// is no need for a key->delegate zone. We still need the delegate->key
// edge for the enterWeakMarkingMode case described above.
if (!key->asTenured().isMarkedBlack()) {
if (!key->zone()->addSweepGroupEdgeTo(delegateZone)) {
return false;
}
}
}
return true;
}

View File

@ -12,6 +12,7 @@
#include "gc/Barrier.h"
#include "gc/DeletePolicy.h"
#include "gc/Tracer.h"
#include "gc/Zone.h"
#include "gc/ZoneAllocator.h"
#include "js/HashTable.h"
@ -47,13 +48,24 @@ bool CheckWeakMapEntryMarking(const WeakMapBase* map, Cell* key, Cell* value);
// the implicit edges stored in the map) and of removing (sweeping) table
// entries when collection is complete.
typedef HashSet<WeakMapBase*, DefaultHasher<WeakMapBase*>, SystemAllocPolicy>
WeakMapSet;
using WeakMapColors = HashMap<WeakMapBase*, js::gc::CellColor,
DefaultHasher<WeakMapBase*>, SystemAllocPolicy>;
// Common base class for all WeakMap specializations, used for calling
// subclasses' GC-related methods.
class WeakMapBase : public mozilla::LinkedListElement<WeakMapBase> {
public:
friend class js::GCMarker;
using CellColor = js::gc::CellColor;
protected:
template <typename T>
CellColor getCellColor(const T& k) const {
if (!k->zone()->shouldMarkInZone() || !k->isTenured()) {
return CellColor::Black;
}
return GetCellColor(k);
}
public:
WeakMapBase(JSObject* memOf, JS::Zone* zone);
@ -83,23 +95,25 @@ class WeakMapBase : public mozilla::LinkedListElement<WeakMapBase> {
// entries of live weak maps whose keys are dead.
static void sweepZone(JS::Zone* zone);
// Sweep the marked weak maps in a zone, updating moved keys.
static void sweepZoneAfterMinorGC(JS::Zone* zone);
// Trace all weak map bindings. Used by the cycle collector.
static void traceAllMappings(WeakMapTracer* tracer);
// Save information about which weak maps are marked for a zone.
static bool saveZoneMarkedWeakMaps(JS::Zone* zone,
WeakMapSet& markedWeakMaps);
WeakMapColors& markedWeakMaps);
// Restore information about which weak maps are marked for many zones.
static void restoreMarkedWeakMaps(WeakMapSet& markedWeakMaps);
static void restoreMarkedWeakMaps(WeakMapColors& markedWeakMaps);
#if defined(JS_GC_ZEAL) || defined(DEBUG)
static bool checkMarkingForZone(JS::Zone* zone);
#endif
static JSObject* getDelegate(JSObject* key);
static JSObject* getDelegate(JSScript* script);
static JSObject* getDelegate(LazyScript* script);
template <typename T>
static JSObject* getDelegate(const T& key);
protected:
// Instance member functions called by the above. Instantiations of WeakMap
@ -115,6 +129,12 @@ class WeakMapBase : public mozilla::LinkedListElement<WeakMapBase> {
virtual void markEntry(GCMarker* marker, gc::Cell* markedCell,
gc::Cell* l) = 0;
// An unmarked CCW with a delegate will add a weakKeys entry for the
// delegate. If the delegate is removed with NukeCrossCompartmentWrapper,
// then the (former) CCW needs to be added to weakKeys instead.
virtual void postSeverDelegate(GCMarker* marker, gc::Cell* key,
Compartment* comp) = 0;
virtual bool markEntries(GCMarker* marker) = 0;
#ifdef JS_GC_ZEAL
@ -132,10 +152,21 @@ class WeakMapBase : public mozilla::LinkedListElement<WeakMapBase> {
// Whether this object has been marked during garbage collection and which
// color it was marked.
bool marked;
gc::MarkColor markColor;
gc::CellColor markColor;
};
namespace detail {
template <typename T>
struct RemoveBarrier {};
template <typename T>
struct RemoveBarrier<js::HeapPtr<T>> {
using Type = T;
};
} // namespace detail
template <class Key, class Value>
class WeakMap
: private HashMap<Key, Value, MovableCellHasher<Key>, ZoneAllocPolicy>,
@ -161,6 +192,8 @@ class WeakMap
// Resolve ambiguity with LinkedListElement<>::remove.
using Base::remove;
using UnbarrieredKey = typename detail::RemoveBarrier<Key>::Type;
explicit WeakMap(JSContext* cx, JSObject* memOf = nullptr);
// Add a read barrier to prevent an incorrectly gray value from escaping the
@ -181,34 +214,112 @@ class WeakMap
return p;
}
template <typename KeyInput, typename ValueInput>
MOZ_MUST_USE bool put(KeyInput&& key, ValueInput&& value) {
MOZ_ASSERT(key);
return Base::put(std::forward<KeyInput>(key),
std::forward<ValueInput>(value));
void remove(Ptr p) {
MOZ_ASSERT(p.found());
if (markColor != CellColor::White) {
forgetKey(p->key());
}
Base::remove(p);
}
void remove(const Lookup& l) {
if (Ptr p = lookup(l)) {
remove(p);
}
}
void clear() {
Base::clear();
JSRuntime* rt = zone()->runtimeFromMainThread();
if (zone()->needsIncrementalBarrier()) {
rt->gc.marker.forgetWeakMap(this, zone());
}
}
template <typename KeyInput, typename ValueInput>
MOZ_MUST_USE bool putNew(KeyInput&& key, ValueInput&& value) {
MOZ_ASSERT(key);
return Base::putNew(std::forward<KeyInput>(key),
std::forward<ValueInput>(value));
MOZ_MUST_USE bool add(AddPtr& p, KeyInput&& k, ValueInput&& v) {
MOZ_ASSERT(k);
if (!Base::add(p, std::forward<KeyInput>(k), std::forward<ValueInput>(v))) {
return false;
}
barrierForInsert(p->key(), p->value());
return true;
}
template <typename KeyInput, typename ValueInput>
MOZ_MUST_USE bool relookupOrAdd(AddPtr& ptr, KeyInput&& key,
ValueInput&& value) {
MOZ_ASSERT(key);
return Base::relookupOrAdd(ptr, std::forward<KeyInput>(key),
std::forward<ValueInput>(value));
MOZ_MUST_USE bool relookupOrAdd(AddPtr& p, KeyInput&& k, ValueInput&& v) {
MOZ_ASSERT(k);
if (!Base::relookupOrAdd(p, std::forward<KeyInput>(k),
std::forward<ValueInput>(v))) {
return false;
}
barrierForInsert(p->key(), p->value());
return true;
}
template <typename KeyInput, typename ValueInput>
MOZ_MUST_USE bool put(KeyInput&& k, ValueInput&& v) {
MOZ_ASSERT(k);
AddPtr p = lookupForAdd(k);
if (p) {
p->value() = std::forward<ValueInput>(v);
return true;
}
return add(p, std::forward<KeyInput>(k), std::forward<ValueInput>(v));
}
template <typename KeyInput, typename ValueInput>
MOZ_MUST_USE bool putNew(KeyInput&& k, ValueInput&& v) {
MOZ_ASSERT(k);
barrierForInsert(k, v);
return Base::putNew(std::forward<KeyInput>(k), std::forward<ValueInput>(v));
}
template <typename KeyInput, typename ValueInput>
void putNewInfallible(KeyInput&& k, ValueInput&& v) {
MOZ_ASSERT(k);
barrierForInsert(k, v);
Base::putNewInfallible(std::forward(k), std::forward<KeyInput>(k));
}
void markEntry(GCMarker* marker, gc::Cell* markedCell,
gc::Cell* origKey) override;
// 'key' has lost its delegate, update our weak key state.
void postSeverDelegate(GCMarker* marker, gc::Cell* key,
Compartment* comp) override;
void trace(JSTracer* trc) override;
protected:
void forgetKey(UnbarrieredKey key) {
// Remove the key or its delegate from weakKeys.
JSRuntime* rt = zone()->runtimeFromMainThread();
if (rt->gc.isIncrementalGCInProgress()) {
if (JSObject* delegate = getDelegate(key)) {
js::gc::WeakKeyTable& weakKeys = delegate->zone()->gcWeakKeys(delegate);
rt->gc.marker.forgetWeakKey(weakKeys, this, delegate, key);
} else {
js::gc::WeakKeyTable& weakKeys = key->zone()->gcWeakKeys(key);
rt->gc.marker.forgetWeakKey(weakKeys, this, key, key);
}
}
}
void barrierForInsert(Key k, const Value& v) {
if (markColor == CellColor::White) {
return;
}
if (!zone()->needsIncrementalBarrier()) {
return;
}
JSTracer* trc = zone()->barrierTracer();
Value tmp = v;
TraceEdge(trc, &tmp, "weakmap inserted value");
MOZ_ASSERT(tmp == v);
}
// We have a key that, if it or its delegate is marked, may lead to a WeakMap
// value getting marked. Insert it or its delegate (if any) into the
// appropriate zone's gcWeakKeys or gcNurseryWeakKeys.
@ -234,9 +345,9 @@ class WeakMap
JS::ExposeObjectToActiveJS(obj);
}
bool keyNeedsMark(GCMarker* marker, JSObject* key) const;
bool keyNeedsMark(GCMarker* marker, JSScript* script) const;
bool keyNeedsMark(GCMarker* marker, LazyScript* script) const;
CellColor getDelegateColor(JSObject* key) const;
CellColor getDelegateColor(JSScript* script) const;
CellColor getDelegateColor(LazyScript* script) const;
bool findZoneEdges() override {
// This is overridden by ObjectValueMap.

View File

@ -399,6 +399,13 @@ void Zone::discardJitCode(FreeOp* fop,
jitZone()->cfgSpace()->lifoAlloc().freeAll();
}
void JS::Zone::delegatePreWriteBarrierInternal(JSObject* obj,
JSObject* delegate) {
MOZ_ASSERT(js::WeakMapBase::getDelegate(obj) == delegate);
MOZ_ASSERT(needsIncrementalBarrier());
GCMarker::fromTracer(barrierTracer())->severWeakDelegate(obj, delegate);
}
#ifdef JSGC_HASH_TABLE_CHECKS
void JS::Zone::checkUniqueIdTableAfterMovingGC() {
for (auto r = uniqueIds().all(); !r.empty(); r.popFront()) {

View File

@ -404,6 +404,12 @@ class Zone : public js::ZoneAllocator, public js::gc::GraphNodeBase<JS::Zone> {
weakCaches().insertBack(cachep);
}
void delegatePreWriteBarrier(JSObject* obj, JSObject* delegate) {
if (needsIncrementalBarrier()) {
delegatePreWriteBarrierInternal(obj, delegate);
}
}
private:
/*
* Mapping from not yet marked keys to a vector of all values that the key
@ -414,9 +420,14 @@ class Zone : public js::ZoneAllocator, public js::gc::GraphNodeBase<JS::Zone> {
js::ZoneOrGCTaskData<js::gc::WeakKeyTable> gcNurseryWeakKeys_;
public:
void delegatePreWriteBarrierInternal(JSObject* obj, JSObject* delegate);
js::gc::WeakKeyTable& gcWeakKeys() { return gcWeakKeys_.ref(); }
js::gc::WeakKeyTable& gcNurseryWeakKeys() { return gcNurseryWeakKeys_.ref(); }
js::gc::WeakKeyTable& gcWeakKeys(const js::gc::Cell* cell) {
return cell->isTenured() ? gcWeakKeys() : gcNurseryWeakKeys();
}
// A set of edges from this zone to other zones used during GC to calculate
// sweep groups.
NodeSet& gcSweepGroupEdges() {

View File

@ -27,6 +27,7 @@
#include "gc/GC-inl.h"
#include "gc/Marking-inl.h"
#include "gc/WeakMap-inl.h"
#include "vm/JSAtom-inl.h"
#include "vm/JSFunction-inl.h"
#include "vm/JSObject-inl.h"
@ -76,6 +77,18 @@ bool Compartment::putWrapper(JSContext* cx, const CrossCompartmentKey& wrapped,
return true;
}
void Compartment::removeWrapper(js::WrapperMap::Ptr p) {
if (p->key().is<JSObject*>()) {
JSObject* key = p->key().as<JSObject*>();
JS::Value value = p->value().unbarrieredGet();
if (js::WeakMapBase::getDelegate(&value.toObject()) == key) {
key->zone()->delegatePreWriteBarrier(&value.toObject(), key);
}
}
crossCompartmentWrappers.remove(p);
}
static JSString* CopyStringPure(JSContext* cx, JSString* str) {
/*
* Directly allocate the copy in the destination compartment, rather than

View File

@ -578,6 +578,8 @@ class JS::Compartment {
const js::CrossCompartmentKey& wrapped,
const js::Value& wrapper);
void removeWrapper(js::WrapperMap::Ptr p);
js::WrapperMap::Ptr lookupWrapper(const js::Value& wrapped) const {
return crossCompartmentWrappers.lookup(js::CrossCompartmentKey(wrapped));
}
@ -586,10 +588,6 @@ class JS::Compartment {
return crossCompartmentWrappers.lookup(js::CrossCompartmentKey(obj));
}
void removeWrapper(js::WrapperMap::Ptr p) {
crossCompartmentWrappers.remove(p);
}
bool hasNurseryAllocatedWrapperEntries(const js::CompartmentFilter& f) {
return crossCompartmentWrappers.hasNurseryAllocatedWrapperEntries(f);
}

View File

@ -12,6 +12,7 @@
#include "vm/Realm.h"
#include "gc/ObjectKind-inl.h"
#include "gc/WeakMap-inl.h"
#include "vm/JSObject-inl.h"
#include "vm/TypeInference-inl.h"
@ -151,6 +152,15 @@ inline void ProxyObject::setPrivate(const Value& priv) {
}
void ProxyObject::nuke() {
// Notify the zone that a delegate is no longer a delegate. Be careful not to
// expose this pointer, because it has already been removed from the wrapper
// map yet we have assertions during tracing that will verify that it is
// still present.
JSObject* delegate = UncheckedUnwrapWithoutExpose(this);
if (delegate != this) {
delegate->zone()->delegatePreWriteBarrier(this, delegate);
}
// Clear the target reference and replaced it with a value that encodes
// various information about the original target.
setSameCompartmentPrivate(DeadProxyTargetValue(this));

View File

@ -488,6 +488,8 @@ class JS::Realm : public JS::shadow::Realm {
// can easily lead to races. Use this method very carefully.
JSRuntime* runtimeFromAnyThread() const { return runtime_; }
void removeWrapper(js::WrapperMap::Ptr p);
const JS::RealmCreationOptions& creationOptions() const {
return creationOptions_;
}