recompiler: avoid overallocation in code buffer (#1247)

sljit performs a single (potentially) oversized executable memory
allocation before generating code. Instead of reserving the requested
amount of space, we can reserve only the used amount and eliminate some
dead space between code blocks.

In some cases sljit may write extra data beyond the generated code.
Because of this, it is no longer safe to assume the remainder of the
code cache is zero initialized. In practice, this assumption only
affected pools which can easily zeroed on demand. This also saves time
on code cache flushes, as we no longer zero the entire cache at once.

ares/component/processor/sh2/recompiler.cpp

@@ -43,6 +43,7 @@ auto SH2::Recompiler::pool(u32 address) -> Pool* {
   if(!pool) {
     pool = (Pool*)allocator.acquire(sizeof(Pool));
     memory::jitprotect(false);
+    *pool = {};
     pool->generation = generation;
     memory::jitprotect(true);
   } else if(address >> 29 == Area::Cached && pool->generation != generation) {
@@ -115,9 +116,7 @@ auto SH2::Recompiler::hash(u32 address, u8 size) -> u64 {
 auto SH2::Recompiler::emit(u32 address) -> Block* {
   if(unlikely(allocator.available() < 1_MiB)) {
     print("SH2 allocator flush\n");
-    memory::jitprotect(false);
-    allocator.release(bump_allocator::zero_fill);
-    memory::jitprotect(true);
+    allocator.release();
     reset();
   }
 

ares/component/processor/sh2/sh2.cpp

@@ -66,9 +66,7 @@ auto SH2::power(bool reset) -> void {
   if constexpr(Accuracy::Recompiler) {
     if(!reset) {
       auto buffer = ares::Memory::FixedAllocator::get().tryAcquire(64_MiB);
-      memory::jitprotect(false);
-      recompiler.allocator.resize(64_MiB, bump_allocator::executable | bump_allocator::zero_fill, buffer);
-      memory::jitprotect(true);
+      recompiler.allocator.resize(64_MiB, bump_allocator::executable, buffer);
     }
     recompiler.reset();
   }

ares/n64/cpu/cpu.cpp

@@ -169,9 +169,7 @@ auto CPU::power(bool reset) -> void {
 
   if constexpr(Accuracy::CPU::Recompiler) {
     auto buffer = ares::Memory::FixedAllocator::get().tryAcquire(64_MiB);
-    memory::jitprotect(false);
-    recompiler.allocator.resize(64_MiB, bump_allocator::executable | bump_allocator::zero_fill, buffer);
-    memory::jitprotect(true);
+    recompiler.allocator.resize(64_MiB, bump_allocator::executable, buffer);
     recompiler.reset();
   }
 }

ares/n64/cpu/recompiler.cpp

@@ -1,6 +1,11 @@
 auto CPU::Recompiler::pool(u32 address) -> Pool* {
   auto& pool = pools[address >> 8 & 0x1fffff];
-  if(!pool) pool = (Pool*)allocator.acquire(sizeof(Pool));
+  if(!pool) {
+    pool = (Pool*)allocator.acquire(sizeof(Pool));
+    memory::jitprotect(false);
+    *pool = {};
+    memory::jitprotect(true);
+  }
   return pool;
 }
 
@@ -21,9 +26,7 @@ auto CPU::Recompiler::fastFetchBlock(u32 address) -> Block* {
 auto CPU::Recompiler::emit(u32 vaddr, u32 address, bool singleInstruction) -> Block* {
   if(unlikely(allocator.available() < 1_MiB)) {
     print("CPU allocator flush\n");
-    memory::jitprotect(false);
-    allocator.release(bump_allocator::zero_fill);
-    memory::jitprotect(true);
+    allocator.release();
     reset();
   }
 

ares/n64/rsp/recompiler.cpp

@@ -61,9 +61,7 @@ auto RSP::Recompiler::block(u12 address) -> Block* {
 auto RSP::Recompiler::emit(u12 address) -> Block* {
   if(unlikely(allocator.available() < 1_MiB)) {
     print("RSP allocator flush\n");
-    memory::jitprotect(false);
-    allocator.release(bump_allocator::zero_fill);
-    memory::jitprotect(true);
+    allocator.release();
     reset();
   }
 

ares/n64/rsp/rsp.cpp

@@ -142,9 +142,7 @@ auto RSP::power(bool reset) -> void {
 
   if constexpr(Accuracy::RSP::Recompiler) {
     auto buffer = ares::Memory::FixedAllocator::get().tryAcquire(64_MiB);
-    memory::jitprotect(false);
-    recompiler.allocator.resize(64_MiB, bump_allocator::executable | bump_allocator::zero_fill, buffer);
-    memory::jitprotect(true);
+    recompiler.allocator.resize(64_MiB, bump_allocator::executable, buffer);
     recompiler.reset();
   }
 

ares/ps1/cpu/cpu.cpp

@@ -265,9 +265,7 @@ auto CPU::power(bool reset) -> void {
 
   if constexpr(Accuracy::CPU::Recompiler) {
     auto buffer = ares::Memory::FixedAllocator::get().tryAcquire(64_MiB);
-    memory::jitprotect(false);
-    recompiler.allocator.resize(64_MiB, bump_allocator::executable | bump_allocator::zero_fill, buffer);
-    memory::jitprotect(true);
+    recompiler.allocator.resize(64_MiB, bump_allocator::executable, buffer);
     recompiler.reset();
   }
 }

ares/ps1/cpu/recompiler.cpp

@@ -1,6 +1,11 @@
 auto CPU::Recompiler::pool(u32 address) -> Pool* {
   auto& pool = pools[address >> 8 & 0x1fffff];
-  if(!pool) pool = (Pool*)allocator.acquire(sizeof(Pool));
+  if(!pool) {
+    pool = (Pool*)allocator.acquire(sizeof(Pool));
+    memory::jitprotect(false);
+    *pool = {};
+    memory::jitprotect(true);
+  }
   return pool;
 }
 
@@ -15,9 +20,7 @@ auto CPU::Recompiler::block(u32 address) -> Block* {
 auto CPU::Recompiler::emit(u32 address) -> Block* {
   if(unlikely(allocator.available() < 1_MiB)) {
     print("CPU allocator flush\n");
-    memory::jitprotect(false);
-    allocator.release(bump_allocator::zero_fill);
-    memory::jitprotect(true);
+    allocator.release();
     reset();
   }
 

nall/bump-allocator.hpp

@@ -85,9 +85,9 @@ struct bump_allocator {
     _offset = nextOffset(size);  //alignment
   }
 
-  auto tryAcquire(u32 size) -> u8* {
+  auto tryAcquire(u32 size, bool reserve = true) -> u8* {
     if((nextOffset(size)) > _capacity) return nullptr;
-    return acquire(size);
+    return reserve ? acquire(size) : acquire();
   }
 
 private:

nall/recompiler/generic/generic.hpp

@@ -31,6 +31,7 @@ namespace nall::recompiler {
 
     auto endFunction() -> u8* {
       u8* code = (u8*)sljit_generate_code(compiler);
+      allocator.reserve(sljit_get_generated_code_size(compiler));
       resetCompiler();
       return code;
     }

thirdparty/sljitAllocator.cpp

@@ -5,5 +5,5 @@
 
 auto sljit_nall_malloc_exec(sljit_uw size, void* exec_allocator_data) -> void* {
   auto allocator = (nall::bump_allocator*)exec_allocator_data;
-  return allocator->acquire(size);
+  return allocator->tryAcquire(size, false);
 }