Memory map allocator implementation

Two memory blocks can be allocated: default region size and huge object size

https://gitee.com/openharmony/ark_js_runtime/issues/I534B6?from=project-issue

Change-Id: I734213cfe43fc04638d4fd97d4cf4c04f7485875
Signed-off-by: yingguofeng@huawei.com <yingguofeng@huawei.com>

BUILD.gn

@@ -443,6 +443,7 @@ ecma_source = [
   "ecmascript/mem/linear_space.cpp",
   "ecmascript/mem/machine_code.cpp",
   "ecmascript/mem/mem_controller.cpp",
+  "ecmascript/mem/mem_map_allocator.cpp",
   "ecmascript/mem/mix_gc.cpp",
   "ecmascript/mem/native_area_allocator.cpp",
   "ecmascript/mem/parallel_work_helper.cpp",

ecmascript/ecma_vm.cpp

@@ -138,6 +138,7 @@ bool EcmaVM::Initialize()
 #endif
     auto globalConst = const_cast<GlobalEnvConstants *>(thread_->GlobalConstants());
     regExpParserCache_ = new RegExpParserCache();
+    MemMapAllocator::GetInstance()->Initialize(options_.TotalSpaceCapacity(), DEFAULT_REGION_SIZE);
     heap_ = new Heap(this);
     heap_->Initialize();
     gcStats_ = chunk_.New<GCStats>(heap_);

ecmascript/js_runtime_options.h

@@ -241,6 +241,11 @@ public:
         return (static_cast<uint32_t>(arkProperties_.GetValue()) & ArkProperties::THREAD_CHECK) != 0;
     }
 
+    size_t TotalSpaceCapacity() const
+    {
+        return totalSpaceCapacity_.GetValue();
+    }
+
     size_t MaxSemiSpaceCapacity() const
     {
         return maxSemiSpaceCapacity_.GetValue();
@@ -539,6 +544,9 @@ private:
         R"(stub aot compiler target triple.
         Possible values: ["x86_64-unknown-linux-gnu", "arm-unknown-linux-gnu", "aarch64-unknown-linux-gnu"].
         Default: "x86_64-unknown-linux-gnu")"};
+    PandArg<size_t> totalSpaceCapacity_ {"totalSpaceCapacity",
+        512 * 1024 * 1024,
+        R"(set total space capacity)"};
     PandArg<size_t> maxSemiSpaceCapacity_ {"maxSemiSpaceCapacity",
         16 * 1024 * 1024,
         R"(set max semi space capacity)"};

ecmascript/mem/heap-inl.h

@@ -28,6 +28,7 @@
 #include "ecmascript/mem/sparse_space.h"
 #include "ecmascript/mem/tagged_object.h"
 #include "ecmascript/mem/barriers-inl.h"
+#include "ecmascript/mem/mem_map_allocator.h"
 
 namespace panda::ecmascript {
 template<class Callback>

ecmascript/mem/heap_region_allocator.cpp

@@ -17,19 +17,9 @@
 #include "ecmascript/mem/heap.h"
 #include "ecmascript/mem/mark_stack.h"
 #include "ecmascript/mem/region.h"
+#include "ecmascript/mem/mem_map_allocator.h"
 #include "libpandabase/mem/pool_manager.h"
 
-#ifdef PANDA_TARGET_UNIX
-#include <sys/prctl.h>
-#ifndef PR_SET_VMA
-#define PR_SET_VMA 0x53564d41
-#endif
-
-#ifndef PR_SET_VMA_ANON_NAME
-#define PR_SET_VMA_ANON_NAME 0
-#endif
-#endif // PANDA_TARGET_UNIX
-
 namespace panda::ecmascript {
 Region *HeapRegionAllocator::AllocateAlignedRegion(Space *space, size_t capacity)
 {
@@ -37,16 +27,14 @@ Region *HeapRegionAllocator::AllocateAlignedRegion(Space *space, size_t capacity
         LOG_ECMA_MEM(FATAL) << "capacity must have a size bigger than 0";
         UNREACHABLE();
     }
-    auto pool = PoolManager::GetMmapMemPool()->AllocPool(capacity, panda::SpaceType::SPACE_TYPE_OBJECT,
-                                                         AllocatorType::RUNSLOTS_ALLOCATOR, nullptr);
+    RegionFlags flags = space->GetRegionFlag();
+    bool isRegular = (flags == RegionFlags::IS_HUGE_OBJECT) ? false : true;
+    auto pool = MemMapAllocator::GetInstance()->Allocate(capacity, DEFAULT_REGION_SIZE, isRegular);
     void *mapMem = pool.GetMem();
     if (mapMem == nullptr) {
         LOG_ECMA_MEM(FATAL) << "pool is empty " << annoMemoryUsage_.load(std::memory_order_relaxed);
         UNREACHABLE();
     }
-#ifdef PANDA_TARGET_UNIX
-    prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, mapMem, pool.GetSize(), "Arkjs Heap");
-#endif // PANDA_TARGET_UNIX
 #if ECMASCRIPT_ENABLE_ZAP_MEM
     if (memset_s(mapMem, capacity, 0, capacity) != EOK) {
         LOG_ECMA(FATAL) << "memset_s failed";
@@ -77,9 +65,7 @@ void HeapRegionAllocator::FreeRegion(Region *region)
         UNREACHABLE();
     }
 #endif
-    PoolManager::GetMmapMemPool()->FreePool(ToVoidPtr(region->GetAllocateBase()), size);
-#ifdef PANDA_TARGET_UNIX
-    prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, region->GetAllocateBase(), size, nullptr);
-#endif // PANDA_TARGET_UNIX
+    bool isRegular = region->InHugeObjectGeneration() ? false : true;
+    MemMapAllocator::GetInstance()->Free(ToVoidPtr(region->GetAllocateBase()), size, isRegular);
 }
 }  // namespace panda::ecmascript

ecmascript/mem/mem.h

@@ -75,8 +75,6 @@ static constexpr size_t MIN_CHUNK_AREA_SIZE = 4 * 1024;
 static constexpr size_t MAX_CACHED_CHUNK_AREA_SIZE = 16 * 1024;
 static constexpr size_t MAX_CHUNK_AREA_SIZE = 1 * 1024 * 1024;
 
-static constexpr uintptr_t PANDA_32BITS_HEAP_START_ADDRESS_256 = 256_KB;
-
 template<typename T>
 constexpr inline bool IsAligned(T value, size_t alignment)
 {

ecmascript/mem/mem_map_allocator.cpp

@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2022 Huawei Device Co., Ltd.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "ecmascript/mem/mem_map_allocator.h"
+
+namespace panda::ecmascript {
+MemMap MemMapAllocator::Allocate(size_t size, size_t alignment, bool isRegular)
+{
+    if (UNLIKELY(memMapTotalSize_ + size > capacity_)) {
+        LOG(ERROR, RUNTIME) << "mem map overflow";
+        return MemMap();
+    }
+    MemMap mem;
+    if (isRegular) {
+        mem = memMapPool_.GetMemFromCache(size);
+        if (mem.GetMem() != nullptr) {
+            memMapTotalSize_ += size;
+            PageTag(mem.GetMem(), size);
+            return mem;
+        }
+        mem = PageMap(REGULAR_REGION_MMAP_SIZE, alignment);
+        mem = memMapPool_.SplitMemToCache(mem);
+    } else {
+        mem = memMapFreeList_.GetMemFromList(size);
+    }
+    if (mem.GetMem() != nullptr) {
+        PageTag(mem.GetMem(), mem.GetSize());
+        memMapTotalSize_ += mem.GetSize();
+    }
+    return mem;
+}
+
+void MemMapAllocator::Free(void *mem, size_t size, bool isRegular)
+{
+    memMapTotalSize_ -= size;
+    PageRelease(mem, size);
+    if (isRegular) {
+        memMapPool_.AddMemToCache(mem, size);
+    } else {
+        memMapFreeList_.AddMemToList(MemMap(mem, size));
+    }
+}
+
+MemMap MemMapAllocator::PageMap(size_t size, size_t alignment)
+{
+    [[maybe_unused]]size_t allocSize = size + alignment;
+    void *result = mmap(0, allocSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    LOG_IF(result == nullptr, FATAL, ECMASCRIPT);
+    auto alignResult = AlignUp(reinterpret_cast<uintptr_t>(result), alignment);
+    size_t leftSize = alignResult - reinterpret_cast<uintptr_t>(result);
+    size_t rightSize = alignment - leftSize;
+    void *alignEndResult = reinterpret_cast<void *>(alignResult + size);
+    munmap(result, leftSize);
+    munmap(alignEndResult, rightSize);
+    return MemMap(reinterpret_cast<void *>(alignResult), size);
+}
+}  // namespace panda::ecmascript

ecmascript/mem/mem_map_allocator.h

@@ -0,0 +1,202 @@
+/*
+ * Copyright (c) 2022 Huawei Device Co., Ltd.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ECMASCRIPT_MEM_MEM_MAP_ALLOCATOR_H
+#define ECMASCRIPT_MEM_MEM_MAP_ALLOCATOR_H
+
+#include <deque>
+#include <map>
+
+#include "ecmascript/mem/mem.h"
+#include "os/mutex.h"
+
+#ifdef PANDA_TARGET_UNIX
+#include <sys/prctl.h>
+#ifndef PR_SET_VMA
+#define PR_SET_VMA 0x53564d41
+#endif
+
+#ifndef PR_SET_VMA_ANON_NAME
+#define PR_SET_VMA_ANON_NAME 0
+#endif
+#endif // PANDA_TARGET_UNIX
+
+namespace panda::ecmascript {
+class MemMap {
+public:
+    MemMap() : mem_(nullptr), size_(0) {}
+    MemMap(void *mem, size_t size) : mem_(mem), size_(size) {};
+
+    inline void *GetMem()
+    {
+        return mem_;
+    }
+
+    inline size_t GetSize()
+    {
+        return size_;
+    }
+private:
+    void *mem_;
+    size_t size_;
+};
+
+// Regular region with length of DEFAULT_REGION_SIZE(256kb)
+class MemMapPool {
+public:
+    explicit MemMapPool() = default;
+    ~MemMapPool() = default;
+
+    NO_COPY_SEMANTIC(MemMapPool);
+    NO_MOVE_SEMANTIC(MemMapPool);
+
+    MemMap GetMemFromCache([[maybe_unused]]size_t size)
+    {
+        ASSERT(size == REGULAR_MMAP_SIZE);
+        os::memory::LockHolder lock(lock_);
+        if (!memMapCache_.empty()) {
+            MemMap mem = memMapCache_.front();
+            memMapCache_.pop_front();
+            return mem;
+        }
+        return MemMap();
+    }
+
+    void AddMemToCache(void *mem, size_t size)
+    {
+        ASSERT(size == REGULAR_MMAP_SIZE);
+        os::memory::LockHolder lock(lock_);
+        memMapCache_.emplace_back(mem, size);
+    }
+
+    MemMap SplitMemToCache(MemMap memMap)
+    {
+        os::memory::LockHolder lock(lock_);
+        auto remainderMem = reinterpret_cast<uintptr_t>(memMap.GetMem()) + REGULAR_MMAP_SIZE;
+        size_t remainderSize = AlignDown(memMap.GetSize() - REGULAR_MMAP_SIZE, REGULAR_MMAP_SIZE);
+        size_t count = remainderSize / REGULAR_MMAP_SIZE;
+        while (count-- > 0) {
+            memMapCache_.emplace_back(reinterpret_cast<void *>(remainderMem), REGULAR_MMAP_SIZE);
+            remainderMem = remainderMem + REGULAR_MMAP_SIZE;
+        }
+        return MemMap(memMap.GetMem(), REGULAR_MMAP_SIZE);
+    }
+
+private:
+    static constexpr size_t REGULAR_MMAP_SIZE = 256_KB;
+    os::memory::Mutex lock_;
+    std::deque<MemMap> memMapCache_;
+    std::deque<MemMap> freedVMCache_;
+};
+
+// Non regular region with length of DEFAULT_REGION_SIZE(256kb) multiple
+class MemMapFreeList {
+public:
+    MemMapFreeList() = default;
+    ~MemMapFreeList() = default;
+
+    void Initialize(MemMap memMap)
+    {
+        freeList_.insert(std::pair<size_t, MemMap>(memMap.GetSize(), memMap));
+    }
+
+    NO_COPY_SEMANTIC(MemMapFreeList);
+    NO_MOVE_SEMANTIC(MemMapFreeList);
+
+    MemMap GetMemFromList(size_t size)
+    {
+        os::memory::LockHolder lock(lock_);
+        auto iterate = freeList_.lower_bound(size);
+        if (iterate == freeList_.end()) {
+            return MemMap();
+        }
+        freeList_.erase(iterate);
+        MemMap memMap = iterate->second;
+        size_t remainderSize = memMap.GetSize() - size;
+        if (remainderSize >= DEFAULT_REGION_SIZE) {
+            auto next = reinterpret_cast<void *>(reinterpret_cast<uintptr_t>(memMap.GetMem()) + size);
+            freeList_.insert(std::pair<size_t, MemMap>(remainderSize, MemMap(next, remainderSize)));
+        }
+        return MemMap(memMap.GetMem(), size);
+    }
+
+    void AddMemToList(MemMap memMap)
+    {
+        os::memory::LockHolder lock(lock_);
+        freeList_.insert(std::pair<size_t, MemMap>(memMap.GetSize(), memMap));
+    }
+
+private:
+    os::memory::Mutex lock_;
+    std::multimap<size_t, MemMap> freeList_;
+};
+
+class MemMapAllocator {
+public:
+    MemMapAllocator() = default;
+    ~MemMapAllocator() = default;
+
+    NO_COPY_SEMANTIC(MemMapAllocator);
+    NO_MOVE_SEMANTIC(MemMapAllocator);
+
+    void Initialize(size_t capacity, size_t alignment)
+    {
+        memMapTotalSize_ = 0;
+        capacity_ = capacity;
+        MemMap memMap = PageMap(NON_REGULAR_MMAP_SIZE, alignment);
+        PageRelease(memMap.GetMem(), memMap.GetSize());
+        memMapFreeList_.Initialize(memMap);
+    }
+
+    static MemMapAllocator *GetInstance()
+    {
+        static MemMapAllocator vmAllocator_;
+        return &vmAllocator_;
+    }
+
+    MemMap Allocate(size_t size, size_t alignment, bool isRegular);
+
+    void Free(void *mem, size_t size, bool isRegular);
+
+private:
+    static constexpr uintptr_t HEAP_START_ADDRESS = 256_KB;
+    static constexpr size_t REGULAR_REGION_MMAP_SIZE = 4_MB;
+    static constexpr size_t NON_REGULAR_MMAP_SIZE = 512_MB;
+
+    MemMap PageMap(size_t size, size_t alignment);
+
+    void PageRelease(void *mem, size_t size)
+    {
+        madvise(mem, size, MADV_DONTNEED);
+    }
+
+    void PageTag(void *mem, size_t size, bool remove = false)
+    {
+#ifdef PANDA_TARGET_UNIX
+        if (remove) {
+            prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, mem, size, nullptr);
+        } else {
+            prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, mem, size, "ArkJS Heap");
+        }
+#endif // PANDA_TARGET_UNIX
+    }
+
+    MemMapPool memMapPool_;
+    MemMapFreeList memMapFreeList_;
+    std::atomic_size_t memMapTotalSize_ {0};
+    size_t capacity_ {0};
+};
+}  // namespace panda::ecmascript
+#endif  // ECMASCRIPT_MEM_MEM_MAP_ALLOCATOR_H