Bug 1074961 - Part 14: Make the ChunkPool list doubly-linked; r=sfink

--HG--
extra : rebase_source : e2c420ad61a10939a8f8c68ddf5cb4822474fe74

js/src/gc/GCRuntime.h

@@ -53,22 +53,36 @@ class ChunkPool
     size_t count() const { return count_; }
 
     /* Must be called with the GC lock taken. */
-    inline Chunk *get(JSRuntime *rt);
+    Chunk *pop();
 
     /* Must be called either during the GC or with the GC lock taken. */
-    inline void put(Chunk *chunk);
+    void push(Chunk *chunk);
+
+    /* Must be called with the GC lock taken. */
+    Chunk *remove(Chunk *chunk);
+
+#ifdef DEBUG
+    bool contains(Chunk *chunk) const;
+    bool verify() const;
+#endif
 
     class Enum {
       public:
         explicit Enum(ChunkPool &pool) : pool(pool), chunkp(&pool.head_) {}
         bool empty() { return !*chunkp; }
         Chunk *front();
-        inline void popFront();
-        inline void removeAndPopFront();
+        void popFront();
+        void removeAndPopFront();
       private:
         ChunkPool &pool;
         Chunk **chunkp;
     };
+
+  private:
+    // ChunkPool controls external resources with interdependencies on the
+    // JSRuntime and related structs, so must not be copied.
+    ChunkPool(const ChunkPool &) MOZ_DELETE;
+    ChunkPool operator=(const ChunkPool &) MOZ_DELETE;
 };
 
 // Performs extra allocation off the main thread so that when memory is

js/src/jsapi-tests/moz.build

@@ -34,6 +34,7 @@ UNIFIED_SOURCES += [
     'testFuncCallback.cpp',
     'testFunctionProperties.cpp',
     'testGCAllocator.cpp',
+    'testGCChunkPool.cpp',
     'testGCExactRooting.cpp',
     'testGCFinalizeCallback.cpp',
     'testGCHeapPostBarriers.cpp',

js/src/jsapi-tests/testGCChunkPool.cpp

@@ -0,0 +1,70 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+* vim: set ts=8 sts=4 et sw=4 tw=99:
+*/
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/Move.h"
+
+#include "gc/GCRuntime.h"
+#include "gc/Heap.h"
+
+#include "jsapi-tests/tests.h"
+
+BEGIN_TEST(testGCChunkPool)
+{
+    const int N = 10;
+    js::gc::ChunkPool pool;
+
+    // Create.
+    for (int i = 0; i < N; ++i) {
+        js::gc::Chunk *chunk = js::gc::Chunk::allocate(rt);
+        CHECK(chunk);
+        pool.push(chunk);
+    }
+    MOZ_ASSERT(pool.verify());
+
+    // Iterate.
+    uint32_t i = 0;
+    for (js::gc::ChunkPool::Enum e(pool); !e.empty(); e.popFront(), ++i)
+        CHECK(e.front());
+    CHECK(i == pool.count());
+    MOZ_ASSERT(pool.verify());
+
+    // Push/Pop.
+    for (int i = 0; i < N; ++i) {
+        js::gc::Chunk *chunkA = pool.pop();
+        js::gc::Chunk *chunkB = pool.pop();
+        js::gc::Chunk *chunkC = pool.pop();
+        pool.push(chunkA);
+        pool.push(chunkB);
+        pool.push(chunkC);
+    }
+    MOZ_ASSERT(pool.verify());
+
+    // Remove.
+    js::gc::Chunk *chunk = nullptr;
+    int offset = N / 2;
+    for (js::gc::ChunkPool::Enum e(pool); !e.empty(); e.popFront(), --offset) {
+        if (offset == 0) {
+            chunk = pool.remove(e.front());
+            break;
+        }
+    }
+    CHECK(chunk);
+    MOZ_ASSERT(!pool.contains(chunk));
+    MOZ_ASSERT(pool.verify());
+    pool.push(chunk);
+
+    // Destruct.
+    js::AutoLockGC lock(rt);
+    for (js::gc::ChunkPool::Enum e(pool); !e.empty();) {
+        js::gc::Chunk *chunk = e.front();
+        e.removeAndPopFront();
+        js::gc::UnmapPages(chunk, js::gc::ChunkSize);
+    }
+
+    return true;
+}
+END_TEST(testGCChunkPool)

js/src/jsgc.cpp

@@ -664,32 +664,92 @@ FreeChunk(JSRuntime *rt, Chunk *p)
 }
 
 /* Must be called with the GC lock taken. */
-inline Chunk *
-ChunkPool::get(JSRuntime *rt)
+Chunk *
+ChunkPool::pop()
 {
-    Chunk *chunk = head_;
-    if (!chunk) {
-        MOZ_ASSERT(!count_);
+    MOZ_ASSERT(bool(head_) == bool(count_));
+    if (!count_)
         return nullptr;
-    }
-
-    MOZ_ASSERT(count_);
-    head_ = chunk->info.next;
-    --count_;
-    return chunk;
+    return remove(head_);
 }
 
 /* Must be called either during the GC or with the GC lock taken. */
-inline void
-ChunkPool::put(Chunk *chunk)
+void
+ChunkPool::push(Chunk *chunk)
 {
+    MOZ_ASSERT(!chunk->info.next);
+    MOZ_ASSERT(!chunk->info.prevp);
+    MOZ_ASSERT_IF(head_, head_->info.prevp == &head_);
+
     chunk->info.age = 0;
     chunk->info.next = head_;
+    chunk->info.prevp = &head_;
+    if (head_)
+        head_->info.prevp = &chunk->info.next;
     head_ = chunk;
-    count_++;
+    ++count_;
+
+    MOZ_ASSERT(verify());
 }
 
-inline Chunk *
+/* Must be called with the GC lock taken. */
+Chunk *
+ChunkPool::remove(Chunk *chunk)
+{
+    MOZ_ASSERT(count_ > 0);
+    MOZ_ASSERT(contains(chunk));
+
+    *chunk->info.prevp = chunk->info.next;
+    if (chunk->info.next) {
+        MOZ_ASSERT(chunk->info.next->info.prevp == &chunk->info.next);
+        chunk->info.next->info.prevp = chunk->info.prevp;
+    }
+    chunk->info.next = nullptr;
+    chunk->info.prevp = nullptr;
+    --count_;
+
+    MOZ_ASSERT(count_ >= 0);
+    MOZ_ASSERT(verify());
+    return chunk;
+}
+
+#ifdef DEBUG
+bool
+ChunkPool::contains(Chunk *chunk) const
+{
+    Chunk *const *prevp = &head_;
+    Chunk *cursor = head_;
+    while (cursor) {
+        MOZ_ASSERT(cursor->info.prevp == prevp);
+        if (cursor == chunk)
+            return true;
+        prevp = &cursor->info.next;
+        cursor = cursor->info.next;
+    }
+    return false;
+}
+
+bool
+ChunkPool::verify() const
+{
+    uint32_t count = 0;
+    Chunk *const *expected_prevp = &head_;
+    Chunk *cursor = head_;
+    while (cursor) {
+        MOZ_ASSERT(cursor->info.prevp);
+        MOZ_ASSERT(cursor->info.prevp == expected_prevp);
+        MOZ_ASSERT(*cursor->info.prevp == cursor);
+        MOZ_ASSERT_IF(cursor->info.next, cursor->info.next->info.prevp == &cursor->info.next);
+        expected_prevp = &cursor->info.next;
+        cursor = cursor->info.next;
+        ++count;
+    }
+    MOZ_ASSERT(count_ == count);
+    return true;
+}
+#endif
+
+Chunk *
 ChunkPool::Enum::front()
 {
     Chunk *chunk = *chunkp;
@@ -697,14 +757,14 @@ ChunkPool::Enum::front()
     return chunk;
 }
 
-inline void
+void
 ChunkPool::Enum::popFront()
 {
     MOZ_ASSERT(!empty());
     chunkp = &front()->info.next;
 }
 
-inline void
+void
 ChunkPool::Enum::removeAndPopFront()
 {
     MOZ_ASSERT(!empty());
@@ -787,7 +847,7 @@ Chunk::allocate(JSRuntime *rt)
 }
 
 /* Must be called with the GC lock taken. */
-inline void
+void
 GCRuntime::releaseChunk(Chunk *chunk)
 {
     MOZ_ASSERT(chunk);
@@ -840,6 +900,8 @@ Chunk::init(JSRuntime *rt)
 
     /* Initialize the chunk info. */
     info.age = 0;
+    info.next = nullptr;
+    info.prevp = nullptr;
     info.trailer.storeBuffer = nullptr;
     info.trailer.location = ChunkLocationBitTenuredHeap;
     info.trailer.runtime = rt;
@@ -1025,7 +1087,7 @@ GCRuntime::moveChunkToFreePool(Chunk *chunk, const AutoLockGC &lock)
     MOZ_ASSERT(chunk->unused());
     MOZ_ASSERT(chunkSet.has(chunk));
     chunkSet.remove(chunk);
-    emptyChunks(lock).put(chunk);
+    emptyChunks(lock).push(chunk);
 }
 
 inline bool
@@ -1084,7 +1146,7 @@ GCRuntime::pickChunk(const AutoLockGC &lock,
     if (chunk)
         return chunk;
 
-    chunk = emptyChunks(lock).get(rt);
+    chunk = emptyChunks(lock).pop();
     if (!chunk) {
         chunk = Chunk::allocate(rt);
         if (!chunk)
@@ -3654,7 +3716,7 @@ BackgroundAllocTask::run()
             if (!chunk)
                 break;
         }
-        chunkPool_.put(chunk);
+        chunkPool_.push(chunk);
     }
 }