From 2d88aefbba1db90eea579f9a9f53335721ee5e31 Mon Sep 17 00:00:00 2001
From: Elad <18193363+elad335@users.noreply.github.com>
Date: Mon, 3 Aug 2026 01:39:25 +0300
Subject: [PATCH] Implement vm::hle_malloc_allocator

Memory allocator using C-memalign and C-free functions of CellOS.
Safer alternative to page allocator in HLE.
---
 rpcs3/Emu/Cell/Modules/cellOskDialog.cpp | 12 ++------
 rpcs3/Emu/Cell/Modules/sys_libc_.cpp     | 36 ++++++++++++++++++++----
 rpcs3/Emu/Cell/lv2/lv2.cpp               |  6 ++++
 rpcs3/Emu/Memory/vm_var.h                |  7 +++++
 4 files changed, 46 insertions(+), 15 deletions(-)

diff --git a/rpcs3/Emu/Cell/Modules/cellOskDialog.cpp b/rpcs3/Emu/Cell/Modules/cellOskDialog.cpp
index 2aee8bdd73..d48f886527 100644
--- a/rpcs3/Emu/Cell/Modules/cellOskDialog.cpp
+++ b/rpcs3/Emu/Cell/Modules/cellOskDialog.cpp
@@ -1,6 +1,7 @@
 #include "stdafx.h"
 #include "Emu/System.h"
 #include "Emu/Cell/PPUModule.h"
+#include "Emu/Cell/Modules/sysPrxForUser.h"
 #include "Emu/Io/interception.h"
 #include "Emu/Io/Keyboard.h"
 #include "Emu/IdManager.h"
@@ -310,14 +311,7 @@ error_code cellOskDialogLoadAsync(u32 container, vm::ptr<CellOskDialogParam> dia
 
 				sysutil_register_cb([&, length = i, string_to_send = std::move(string_to_send)](ppu_thread& cb_ppu) -> s32
 				{
-					cb_ppu.state += cpu_flag::wait;
-
-					vm::var<u16[], vm::page_allocator<>> string_var(CELL_OSKDIALOG_STRING_SIZE, string_to_send.data());
-
-					if (cb_ppu.check_state())
-					{
-						return 0;
-					}
+					vm::var<u16[], vm::hle_malloc_allocator> string_var(CELL_OSKDIALOG_STRING_SIZE, string_to_send.data());
 
 					const u32 return_value = ccb(cb_ppu, string_var.begin(), static_cast<s32>(length));
 					cellOskDialog.warning("osk_confirm_callback return_value=%d", return_value);
@@ -512,7 +506,7 @@ error_code cellOskDialogLoadAsync(u32 container, vm::ptr<CellOskDialogParam> dia
 			// Prepare callback variables
 			vm::var<CellOskDialogKeyMessage> keyMessage(key_message);
 			vm::var<u32> action(CELL_OSKDIALOG_CHANGE_NO_EVENT);
-			vm::var<u16[], vm::page_allocator<>> pActionInfo(::narrow<u32>(string_to_send.size()), string_to_send.data());
+			vm::var<u16[], vm::hle_malloc_allocator> pActionInfo(::narrow<u32>(string_to_send.size()), string_to_send.data());
 
 			// Create helpers for logging
 			std::u16string utf16_string(reinterpret_cast<const char16_t*>(string_to_send.data()), string_to_send.size());
diff --git a/rpcs3/Emu/Cell/Modules/sys_libc_.cpp b/rpcs3/Emu/Cell/Modules/sys_libc_.cpp
index c5647dc0b2..7fc2d74d15 100644
--- a/rpcs3/Emu/Cell/Modules/sys_libc_.cpp
+++ b/rpcs3/Emu/Cell/Modules/sys_libc_.cpp
@@ -1,6 +1,7 @@
 #include "stdafx.h"
 #include "Emu/Cell/lv2/sys_tty.h"
 #include "Emu/Cell/PPUModule.h"
+#include "Emu/Cell/PPUFunction.h"
 #include "Utilities/cfmt.h"
 
 LOG_CHANNEL(sysPrxForUser);
@@ -367,27 +368,50 @@ vm::cptr<char> _sys_strrchr(vm::cptr<char> str, char ch)
 	return res;
 }
 
-u32 _sys_malloc(u32 size)
+u32 _sys_malloc(ppu_thread& ppu, u32 size)
 {
+	ppu.state += cpu_flag::wait;
+
 	sysPrxForUser.warning("_sys_malloc(size=0x%x)", size);
 
-	return vm::alloc(size, vm::main);
+	const u32 res = vm::alloc(size, vm::main);
+	ppu.check_state();
+	return res;
 }
 
-u32 _sys_memalign(u32 align, u32 size)
+u32 _sys_memalign(ppu_thread& ppu, u32 align, u32 size)
 {
+	ppu.state += cpu_flag::wait;
+
 	sysPrxForUser.warning("_sys_memalign(align=0x%x, size=0x%x)", align, size);
 
-	return vm::alloc(size, vm::main, std::max<u32>(align, 0x10000));
+	const u32 res = vm::alloc(size, vm::main, std::max<u32>(align, 0x10000));
+	ppu.check_state();
+	return res;
 }
 
-error_code _sys_free(u32 addr)
+void _sys_free(ppu_thread& ppu, u32 addr)
 {
+	ppu.state += cpu_flag::wait;
+
 	sysPrxForUser.warning("_sys_free(addr=0x%x)", addr);
 
 	vm::dealloc(addr, vm::main);
 
-	return CELL_OK;
+	ppu.check_state();
+}
+
+std::pair<vm::addr_t, u32> vm::hle_malloc_allocator::alloc(u32 size, u32 align)
+{
+	const auto ppu = ensure(cpu_thread::get_current<ppu_thread>());
+	const u32 addr = vm::cast(ppu_execute<&_sys_memalign>(*ppu, align, size));
+	return { vm::cast(addr), addr ? size : 0 };
+}
+
+void vm::hle_malloc_allocator::dealloc(u32 addr, u32 size) noexcept
+{
+	const auto ppu = ensure(cpu_thread::get_current<ppu_thread>());
+	ppu_execute<&_sys_free>(*ppu, addr);
 }
 
 s32 _sys_snprintf(ppu_thread& ppu, vm::ptr<char> dst, u32 count, vm::cptr<char> fmt, ppu_va_args_t va_args)
diff --git a/rpcs3/Emu/Cell/lv2/lv2.cpp b/rpcs3/Emu/Cell/lv2/lv2.cpp
index c6b81334ae..7cda0052a4 100644
--- a/rpcs3/Emu/Cell/lv2/lv2.cpp
+++ b/rpcs3/Emu/Cell/lv2/lv2.cpp
@@ -1117,6 +1117,12 @@ void fmt_class_string<CellError>::format(std::string& out, u64 arg)
 	});
 }
 
+template <>
+void fmt_class_string<error_code>::format(std::string& out, u64 arg)
+{
+	fmt_class_string<CellError>::format(out, arg);
+}
+
 stx::init_lock acquire_lock(stx::init_mutex& mtx, ppu_thread* ppu)
 {
 	if (!ppu)
diff --git a/rpcs3/Emu/Memory/vm_var.h b/rpcs3/Emu/Memory/vm_var.h
index 4231ab048b..7c9e207b37 100644
--- a/rpcs3/Emu/Memory/vm_var.h
+++ b/rpcs3/Emu/Memory/vm_var.h
@@ -20,6 +20,13 @@ namespace vm
 		}
 	};
 
+	struct hle_malloc_allocator
+	{
+		// Defined externally
+		static std::pair<vm::addr_t, u32> alloc(u32 size, u32 align);
+		static void dealloc(u32 addr, u32 size) noexcept;
+	};
+
 	template <typename T>
 	struct stack_allocator
 	{