diff --git a/src/core/address_space.cpp b/src/core/address_space.cpp index 3f063ea76..422c67e17 100644 --- a/src/core/address_space.cpp +++ b/src/core/address_space.cpp @@ -93,7 +93,10 @@ static u64 BackingSize = ORBIS_KERNEL_TOTAL_MEM_DEV_PRO; struct MemoryRegion { VAddr base; - size_t size; + PAddr phys_base; + u64 size; + u32 prot; + s32 fd; bool is_mapped; }; @@ -159,7 +162,8 @@ struct AddressSpace::Impl { // Restrict region size to avoid overly fragmenting the virtual memory space. if (info.State == MEM_FREE && info.RegionSize > 0x1000000) { VAddr addr = Common::AlignUp(reinterpret_cast(info.BaseAddress), alignment); - regions.emplace(addr, MemoryRegion{addr, size, false}); + regions.emplace(addr, + MemoryRegion{addr, PAddr(-1), size, PAGE_NOACCESS, -1, false}); } } @@ -207,29 +211,32 @@ struct AddressSpace::Impl { ~Impl() { if (virtual_base) { if (!VirtualFree(virtual_base, 0, MEM_RELEASE)) { - LOG_CRITICAL(Render, "Failed to free virtual memory"); + LOG_CRITICAL(Core, "Failed to free virtual memory"); } } if (backing_base) { if (!UnmapViewOfFile2(process, backing_base, MEM_PRESERVE_PLACEHOLDER)) { - LOG_CRITICAL(Render, "Failed to unmap backing memory placeholder"); + LOG_CRITICAL(Core, "Failed to unmap backing memory placeholder"); } if (!VirtualFreeEx(process, backing_base, 0, MEM_RELEASE)) { - LOG_CRITICAL(Render, "Failed to free backing memory"); + LOG_CRITICAL(Core, "Failed to free backing memory"); } } if (!CloseHandle(backing_handle)) { - LOG_CRITICAL(Render, "Failed to free backing memory file handle"); + LOG_CRITICAL(Core, "Failed to free backing memory file handle"); } } - void* Map(VAddr virtual_addr, PAddr phys_addr, size_t size, ULONG prot, uintptr_t fd = 0) { - // Before mapping we must carve a placeholder with the exact properties of our mapping. - auto* region = EnsureSplitRegionForMapping(virtual_addr, size); - region->is_mapped = true; + void* MapRegion(MemoryRegion* region) { + VAddr virtual_addr = region->base; + PAddr phys_addr = region->phys_base; + u64 size = region->size; + ULONG prot = region->prot; + s32 fd = region->fd; + void* ptr = nullptr; if (phys_addr != -1) { - HANDLE backing = fd ? reinterpret_cast(fd) : backing_handle; + HANDLE backing = fd != -1 ? reinterpret_cast(fd) : backing_handle; if (fd && prot == PAGE_READONLY) { DWORD resultvar; ptr = VirtualAlloc2(process, reinterpret_cast(virtual_addr), size, @@ -257,110 +264,136 @@ struct AddressSpace::Impl { return ptr; } - void Unmap(VAddr virtual_addr, size_t size, bool has_backing) { - bool ret; - if (has_backing) { + void UnmapRegion(MemoryRegion* region) { + VAddr virtual_addr = region->base; + PAddr phys_base = region->phys_base; + u64 size = region->size; + + bool ret = false; + if (phys_base != -1) { ret = UnmapViewOfFile2(process, reinterpret_cast(virtual_addr), MEM_PRESERVE_PLACEHOLDER); } else { ret = VirtualFreeEx(process, reinterpret_cast(virtual_addr), size, MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER); } - ASSERT_MSG(ret, "Unmap operation on virtual_addr={:#X} failed: {}", virtual_addr, + ASSERT_MSG(ret, "Unmap on virtual_addr {:#x}, size {:#x} failed: {}", virtual_addr, size, Common::GetLastErrorMsg()); - - // The unmap call will create a new placeholder region. We need to see if we can coalesce it - // with neighbors. - JoinRegionsAfterUnmap(virtual_addr, size); } - // The following code is inspired from Dolphin's MemArena - // https://github.com/dolphin-emu/dolphin/blob/deee3ee4/Source/Core/Common/MemArenaWin.cpp#L212 - MemoryRegion* EnsureSplitRegionForMapping(VAddr address, size_t size) { - // Find closest region that is <= the given address by using upper bound and decrementing - auto it = regions.upper_bound(address); - ASSERT_MSG(it != regions.begin(), "Invalid address {:#x}", address); - --it; - ASSERT_MSG(!it->second.is_mapped, - "Attempt to map {:#x} with size {:#x} which overlaps with {:#x} mapping", - address, size, it->second.base); - auto& [base, region] = *it; + void SplitRegion(VAddr virtual_addr, u64 size) { + // First, get the region this range covers + auto it = std::prev(regions.upper_bound(virtual_addr)); - const VAddr mapping_address = region.base; - const size_t region_size = region.size; - if (mapping_address == address) { - // If this region is already split up correctly we don't have to do anything - if (region_size == size) { - return ®ion; + // All unmapped areas will coalesce, so there should be a region + // containing the full requested range. If not, then something is mapped here. + ASSERT_MSG(it->second.base + it->second.size >= virtual_addr + size, + "Cannot fit region into one placeholder"); + + // If the region is mapped, we need to unmap first before we can modify the placeholders. + if (it->second.is_mapped) { + ASSERT_MSG(it->second.phys_base != -1 || !it->second.is_mapped, + "Cannot split unbacked mapping"); + UnmapRegion(&it->second); + } + + // We need to split this region to create a matching placeholder. + if (it->second.base != virtual_addr) { + // Requested address is not the start of the containing region, + // create a new region to represent the memory before the requested range. + auto& region = it->second; + u64 base_offset = virtual_addr - region.base; + u64 next_region_size = region.size - base_offset; + PAddr next_region_phys_base = -1; + if (region.is_mapped) { + next_region_phys_base = region.phys_base + base_offset; } + region.size = base_offset; - ASSERT_MSG(region_size >= size, - "Region with address {:#x} and size {:#x} can't fit {:#x}", mapping_address, - region_size, size); - - // Split the placeholder. - if (!VirtualFreeEx(process, LPVOID(address), size, + // Use VirtualFreeEx to create the split. + if (!VirtualFreeEx(process, LPVOID(region.base), region.size, MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER)) { UNREACHABLE_MSG("Region splitting failed: {}", Common::GetLastErrorMsg()); - return nullptr; } - // Update tracked mappings and return the first of the two + // If the mapping was mapped, remap the region. + if (region.is_mapped) { + MapRegion(®ion); + } + + // Store a new region matching the removed area. + it = regions.emplace_hint(std::next(it), virtual_addr, + MemoryRegion(virtual_addr, next_region_phys_base, + next_region_size, region.prot, region.fd, + region.is_mapped)); + } + + // At this point, the region's base will match virtual_addr. + // Now check for a size difference. + if (it->second.size != size) { + // The requested size is smaller than the current region placeholder. + // Update region to match the requested region, + // then make a new region to represent the remaining space. + auto& region = it->second; + VAddr next_region_addr = region.base + size; + u64 next_region_size = region.size - size; + PAddr next_region_phys_base = -1; + if (region.is_mapped) { + next_region_phys_base = region.phys_base + size; + } region.size = size; - const VAddr new_mapping_start = address + size; - regions.emplace_hint(std::next(it), new_mapping_start, - MemoryRegion(new_mapping_start, region_size - size, false)); - return ®ion; + + // Store the new region matching the remaining space + regions.emplace_hint(std::next(it), next_region_addr, + MemoryRegion(next_region_addr, next_region_phys_base, + next_region_size, region.prot, region.fd, + region.is_mapped)); + + // Use VirtualFreeEx to create the split. + if (!VirtualFreeEx(process, LPVOID(region.base), region.size, + MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER)) { + UNREACHABLE_MSG("Region splitting failed: {}", Common::GetLastErrorMsg()); + } + + // If these regions were mapped, then map the unmapped area beyond the requested range. + if (region.is_mapped) { + MapRegion(&std::next(it)->second); + } } - ASSERT(mapping_address < address); - - // Is there enough space to map this? - const size_t offset_in_region = address - mapping_address; - const size_t minimum_size = size + offset_in_region; - ASSERT(region_size >= minimum_size); - - // Split the placeholder. - if (!VirtualFreeEx(process, LPVOID(address), size, - MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER)) { - UNREACHABLE_MSG("Region splitting failed: {}", Common::GetLastErrorMsg()); - return nullptr; - } - - // Do we now have two regions or three regions? - if (region_size == minimum_size) { - // Split into two; update tracked mappings and return the second one - region.size = offset_in_region; - it = regions.emplace_hint(std::next(it), address, MemoryRegion(address, size, false)); - return &it->second; - } else { - // Split into three; update tracked mappings and return the middle one - region.size = offset_in_region; - const VAddr middle_mapping_start = address; - const size_t middle_mapping_size = size; - const VAddr after_mapping_start = address + size; - const size_t after_mapping_size = region_size - minimum_size; - it = regions.emplace_hint(std::next(it), after_mapping_start, - MemoryRegion(after_mapping_start, after_mapping_size, false)); - it = regions.emplace_hint( - it, middle_mapping_start, - MemoryRegion(middle_mapping_start, middle_mapping_size, false)); - return &it->second; + // If the requested region was mapped, remap it. + if (it->second.is_mapped) { + MapRegion(&it->second); } } - void JoinRegionsAfterUnmap(VAddr address, size_t size) { - // There should be a mapping that matches the request exactly, find it - auto it = regions.find(address); - ASSERT_MSG(it != regions.end() && it->second.size == size, - "Invalid address/size given to unmap."); + void* Map(VAddr virtual_addr, PAddr phys_addr, u64 size, ULONG prot, s32 fd = -1) { + // Split surrounding regions to create a placeholder + SplitRegion(virtual_addr, size); + + // Get the region this range covers + auto it = std::prev(regions.upper_bound(virtual_addr)); auto& [base, region] = *it; - region.is_mapped = false; + + ASSERT_MSG(!region.is_mapped, "Cannot overwrite mapped region"); + + // Now we have a region matching the requested region, perform the actual mapping. + region.is_mapped = true; + region.phys_base = phys_addr; + region.prot = prot; + region.fd = fd; + return MapRegion(®ion); + } + + void CoalesceFreeRegions(VAddr virtual_addr) { + // First, get the region to update + auto it = std::prev(regions.upper_bound(virtual_addr)); + ASSERT_MSG(!it->second.is_mapped, "Cannot coalesce mapped regions"); // Check if a placeholder exists right before us. auto it_prev = it != regions.begin() ? std::prev(it) : regions.end(); if (it_prev != regions.end() && !it_prev->second.is_mapped) { - const size_t total_size = it_prev->second.size + size; + const u64 total_size = it_prev->second.size + it->second.size; if (!VirtualFreeEx(process, LPVOID(it_prev->first), total_size, MEM_RELEASE | MEM_COALESCE_PLACEHOLDERS)) { UNREACHABLE_MSG("Region coalescing failed: {}", Common::GetLastErrorMsg()); @@ -374,7 +407,7 @@ struct AddressSpace::Impl { // Check if a placeholder exists right after us. auto it_next = std::next(it); if (it_next != regions.end() && !it_next->second.is_mapped) { - const size_t total_size = it->second.size + it_next->second.size; + const u64 total_size = it->second.size + it_next->second.size; if (!VirtualFreeEx(process, LPVOID(it->first), total_size, MEM_RELEASE | MEM_COALESCE_PLACEHOLDERS)) { UNREACHABLE_MSG("Region coalescing failed: {}", Common::GetLastErrorMsg()); @@ -385,7 +418,47 @@ struct AddressSpace::Impl { } } - void Protect(VAddr virtual_addr, size_t size, bool read, bool write, bool execute) { + void Unmap(VAddr virtual_addr, u64 size) { + // Loop through all regions in the requested range + u64 remaining_size = size; + VAddr current_addr = virtual_addr; + while (remaining_size > 0) { + // Get the region containing our current address. + auto it = std::prev(regions.upper_bound(current_addr)); + + // If necessary, split regions to ensure a valid unmap. + // To prevent complication, ensure size is within the bounds of the current region. + u64 base_offset = current_addr - it->second.base; + u64 size_to_unmap = std::min(it->second.size - base_offset, remaining_size); + if (current_addr != it->second.base || size_to_unmap != it->second.size) { + SplitRegion(current_addr, size_to_unmap); + } + + // Repair the region pointer, as SplitRegion modifies the regions map. + it = std::prev(regions.upper_bound(current_addr)); + auto& [base, region] = *it; + + // Unmap the region if it was previously mapped + if (region.is_mapped) { + UnmapRegion(®ion); + } + + // Update region data + region.is_mapped = false; + region.fd = -1; + region.phys_base = -1; + region.prot = PAGE_NOACCESS; + + // Coalesce any free space + CoalesceFreeRegions(current_addr); + + // Update loop variables + remaining_size -= size_to_unmap; + current_addr += size_to_unmap; + } + } + + void Protect(VAddr virtual_addr, u64 size, bool read, bool write, bool execute) { DWORD new_flags{}; if (write && !read) { @@ -415,7 +488,7 @@ struct AddressSpace::Impl { // If no flags are assigned, then something's gone wrong. if (new_flags == 0) { - LOG_CRITICAL(Common_Memory, + LOG_CRITICAL(Core, "Unsupported protection flag combination for address {:#x}, size {}, " "read={}, write={}, execute={}", virtual_addr, size, read, write, execute); @@ -429,8 +502,8 @@ struct AddressSpace::Impl { continue; } const auto& region = it->second; - const size_t range_addr = std::max(region.base, virtual_addr); - const size_t range_size = std::min(region.base + region.size, virtual_end) - range_addr; + const u64 range_addr = std::max(region.base, virtual_addr); + const u64 range_size = std::min(region.base + region.size, virtual_end) - range_addr; DWORD old_flags{}; if (!VirtualProtectEx(process, LPVOID(range_addr), range_size, new_flags, &old_flags)) { UNREACHABLE_MSG( @@ -453,11 +526,11 @@ struct AddressSpace::Impl { u8* backing_base{}; u8* virtual_base{}; u8* system_managed_base{}; - size_t system_managed_size{}; + u64 system_managed_size{}; u8* system_reserved_base{}; - size_t system_reserved_size{}; + u64 system_reserved_size{}; u8* user_base{}; - size_t user_size{}; + u64 user_size{}; std::map regions; }; #else @@ -601,7 +674,7 @@ struct AddressSpace::Impl { } } - void* Map(VAddr virtual_addr, PAddr phys_addr, size_t size, PosixPageProtection prot, + void* Map(VAddr virtual_addr, PAddr phys_addr, u64 size, PosixPageProtection prot, int fd = -1) { m_free_regions.subtract({virtual_addr, virtual_addr + size}); const int handle = phys_addr != -1 ? (fd == -1 ? backing_fd : fd) : -1; @@ -613,10 +686,10 @@ struct AddressSpace::Impl { return ret; } - void Unmap(VAddr virtual_addr, size_t size, bool) { + void Unmap(VAddr virtual_addr, u64 size, bool) { // Check to see if we are adjacent to any regions. - auto start_address = virtual_addr; - auto end_address = start_address + size; + VAddr start_address = virtual_addr; + VAddr end_address = start_address + size; auto it = m_free_regions.find({start_address - 1, end_address + 1}); // If we are, join with them, ensuring we stay in bounds. @@ -634,7 +707,7 @@ struct AddressSpace::Impl { ASSERT_MSG(ret != MAP_FAILED, "mmap failed: {}", strerror(errno)); } - void Protect(VAddr virtual_addr, size_t size, bool read, bool write, bool execute) { + void Protect(VAddr virtual_addr, u64 size, bool read, bool write, bool execute) { int flags = PROT_NONE; if (read) { flags |= PROT_READ; @@ -654,11 +727,11 @@ struct AddressSpace::Impl { int backing_fd; u8* backing_base{}; u8* system_managed_base{}; - size_t system_managed_size{}; + u64 system_managed_size{}; u8* system_reserved_base{}; - size_t system_reserved_size{}; + u64 system_reserved_size{}; u8* user_base{}; - size_t user_size{}; + u64 user_size{}; boost::icl::interval_set m_free_regions; }; #endif @@ -675,8 +748,7 @@ AddressSpace::AddressSpace() : impl{std::make_unique()} { AddressSpace::~AddressSpace() = default; -void* AddressSpace::Map(VAddr virtual_addr, size_t size, u64 alignment, PAddr phys_addr, - bool is_exec) { +void* AddressSpace::Map(VAddr virtual_addr, u64 size, PAddr phys_addr, bool is_exec) { #if ARCH_X86_64 const auto prot = is_exec ? PAGE_EXECUTE_READWRITE : PAGE_READWRITE; #else @@ -687,8 +759,7 @@ void* AddressSpace::Map(VAddr virtual_addr, size_t size, u64 alignment, PAddr ph return impl->Map(virtual_addr, phys_addr, size, prot); } -void* AddressSpace::MapFile(VAddr virtual_addr, size_t size, size_t offset, u32 prot, - uintptr_t fd) { +void* AddressSpace::MapFile(VAddr virtual_addr, u64 size, u64 offset, u32 prot, uintptr_t fd) { #ifdef _WIN32 return impl->Map(virtual_addr, offset, size, ToWindowsProt(std::bit_cast(prot)), fd); @@ -698,31 +769,15 @@ void* AddressSpace::MapFile(VAddr virtual_addr, size_t size, size_t offset, u32 #endif } -void AddressSpace::Unmap(VAddr virtual_addr, size_t size, VAddr start_in_vma, VAddr end_in_vma, - PAddr phys_base, bool is_exec, bool has_backing, bool readonly_file) { +void AddressSpace::Unmap(VAddr virtual_addr, u64 size, bool has_backing) { #ifdef _WIN32 - // There does not appear to be comparable support for partial unmapping on Windows. - // Unfortunately, a least one title was found to require this. The workaround is to unmap - // the entire allocation and remap the portions outside of the requested unmapping range. - impl->Unmap(virtual_addr, size, has_backing && !readonly_file); - - // TODO: Determine if any titles require partial unmapping support for un-backed allocations. - ASSERT_MSG(has_backing || (start_in_vma == 0 && end_in_vma == size), - "Partial unmapping of un-backed allocations is not supported"); - - if (start_in_vma != 0) { - Map(virtual_addr, start_in_vma, 0, phys_base, is_exec); - } - - if (end_in_vma != size) { - Map(virtual_addr + end_in_vma, size - end_in_vma, 0, phys_base + end_in_vma, is_exec); - } + impl->Unmap(virtual_addr, size); #else - impl->Unmap(virtual_addr + start_in_vma, end_in_vma - start_in_vma, has_backing); + impl->Unmap(virtual_addr, size, has_backing); #endif } -void AddressSpace::Protect(VAddr virtual_addr, size_t size, MemoryPermission perms) { +void AddressSpace::Protect(VAddr virtual_addr, u64 size, MemoryPermission perms) { const bool read = True(perms & MemoryPermission::Read); const bool write = True(perms & MemoryPermission::Write); const bool execute = True(perms & MemoryPermission::Execute); diff --git a/src/core/address_space.h b/src/core/address_space.h index 5c50039bd..fa47bb47e 100644 --- a/src/core/address_space.h +++ b/src/core/address_space.h @@ -39,7 +39,7 @@ public: [[nodiscard]] const u8* SystemManagedVirtualBase() const noexcept { return system_managed_base; } - [[nodiscard]] size_t SystemManagedVirtualSize() const noexcept { + [[nodiscard]] u64 SystemManagedVirtualSize() const noexcept { return system_managed_size; } @@ -49,7 +49,7 @@ public: [[nodiscard]] const u8* SystemReservedVirtualBase() const noexcept { return system_reserved_base; } - [[nodiscard]] size_t SystemReservedVirtualSize() const noexcept { + [[nodiscard]] u64 SystemReservedVirtualSize() const noexcept { return system_reserved_size; } @@ -59,7 +59,7 @@ public: [[nodiscard]] const u8* UserVirtualBase() const noexcept { return user_base; } - [[nodiscard]] size_t UserVirtualSize() const noexcept { + [[nodiscard]] u64 UserVirtualSize() const noexcept { return user_size; } @@ -73,17 +73,15 @@ public: * If zero is provided the mapping is considered as private. * @return A pointer to the mapped memory. */ - void* Map(VAddr virtual_addr, size_t size, u64 alignment = 0, PAddr phys_addr = -1, - bool exec = false); + void* Map(VAddr virtual_addr, u64 size, PAddr phys_addr = -1, bool exec = false); /// Memory maps a specified file descriptor. - void* MapFile(VAddr virtual_addr, size_t size, size_t offset, u32 prot, uintptr_t fd); + void* MapFile(VAddr virtual_addr, u64 size, u64 offset, u32 prot, uintptr_t fd); /// Unmaps specified virtual memory area. - void Unmap(VAddr virtual_addr, size_t size, VAddr start_in_vma, VAddr end_in_vma, - PAddr phys_base, bool is_exec, bool has_backing, bool readonly_file); + void Unmap(VAddr virtual_addr, u64 size, bool has_backing); - void Protect(VAddr virtual_addr, size_t size, MemoryPermission perms); + void Protect(VAddr virtual_addr, u64 size, MemoryPermission perms); // Returns an interval set containing all usable regions. boost::icl::interval_set GetUsableRegions(); @@ -93,11 +91,11 @@ private: std::unique_ptr impl; u8* backing_base{}; u8* system_managed_base{}; - size_t system_managed_size{}; + u64 system_managed_size{}; u8* system_reserved_base{}; - size_t system_reserved_size{}; + u64 system_reserved_size{}; u8* user_base{}; - size_t user_size{}; + u64 user_size{}; }; } // namespace Core diff --git a/src/core/devtools/widget/memory_map.cpp b/src/core/devtools/widget/memory_map.cpp index 278c6595c..d1d1eb410 100644 --- a/src/core/devtools/widget/memory_map.cpp +++ b/src/core/devtools/widget/memory_map.cpp @@ -32,7 +32,7 @@ bool MemoryMapViewer::Iterator::DrawLine() { TableNextColumn(); Text("%s", magic_enum::enum_name(m.prot).data()); TableNextColumn(); - if (m.is_exec) { + if (True(m.prot & MemoryProt::CpuExec)) { Text("X"); } TableNextColumn(); @@ -44,7 +44,7 @@ bool MemoryMapViewer::Iterator::DrawLine() { return false; } auto m = dmem.it->second; - if (m.dma_type == DMAType::Free) { + if (m.dma_type == PhysicalMemoryType::Free) { ++dmem.it; return DrawLine(); } @@ -56,7 +56,8 @@ bool MemoryMapViewer::Iterator::DrawLine() { auto type = static_cast<::Libraries::Kernel::MemoryTypes>(m.memory_type); Text("%s", magic_enum::enum_name(type).data()); TableNextColumn(); - Text("%d", m.dma_type == DMAType::Pooled || m.dma_type == DMAType::Committed); + Text("%d", + m.dma_type == PhysicalMemoryType::Pooled || m.dma_type == PhysicalMemoryType::Committed); ++dmem.it; return true; } diff --git a/src/core/devtools/widget/memory_map.h b/src/core/devtools/widget/memory_map.h index cc7697c8c..3bbec4643 100644 --- a/src/core/devtools/widget/memory_map.h +++ b/src/core/devtools/widget/memory_map.h @@ -11,8 +11,8 @@ class MemoryMapViewer { struct Iterator { bool is_vma; struct { - MemoryManager::DMemMap::iterator it; - MemoryManager::DMemMap::iterator end; + MemoryManager::PhysMap::iterator it; + MemoryManager::PhysMap::iterator end; } dmem; struct { MemoryManager::VMAMap::iterator it; diff --git a/src/core/libraries/kernel/memory.cpp b/src/core/libraries/kernel/memory.cpp index 62903ff72..3aec8193a 100644 --- a/src/core/libraries/kernel/memory.cpp +++ b/src/core/libraries/kernel/memory.cpp @@ -102,6 +102,7 @@ s32 PS4_SYSV_ABI sceKernelReleaseDirectMemory(u64 start, u64 len) { if (len == 0) { return ORBIS_OK; } + LOG_INFO(Kernel_Vmm, "called start = {:#x}, len = {:#x}", start, len); auto* memory = Core::Memory::Instance(); memory->Free(start, len); return ORBIS_OK; diff --git a/src/core/memory.cpp b/src/core/memory.cpp index 3a4a16933..4567475cd 100644 --- a/src/core/memory.cpp +++ b/src/core/memory.cpp @@ -5,6 +5,7 @@ #include "common/assert.h" #include "common/config.h" #include "common/debug.h" +#include "common/elf_info.h" #include "core/file_sys/fs.h" #include "core/libraries/kernel/memory.h" #include "core/libraries/kernel/orbis_error.h" @@ -25,6 +26,9 @@ MemoryManager::MemoryManager() { VirtualMemoryArea{region.lower(), region.upper() - region.lower()}); LOG_INFO(Kernel_Vmm, "{:#x} - {:#x}", region.lower(), region.upper()); } + + ASSERT_MSG(Libraries::Kernel::sceKernelGetCompiledSdkVersion(&sdk_version) == 0, + "Failed to get compiled SDK version"); } MemoryManager::~MemoryManager() = default; @@ -55,14 +59,14 @@ void MemoryManager::SetupMemoryRegions(u64 flexible_size, bool use_extended_mem1 // Insert an area that covers the direct memory physical address block. // Note that this should never be called after direct memory allocations have been made. dmem_map.clear(); - dmem_map.emplace(0, DirectMemoryArea{0, total_direct_size}); + dmem_map.emplace(0, PhysicalMemoryArea{0, total_direct_size}); // Insert an area that covers the flexible memory physical address block. // Note that this should never be called after flexible memory allocations have been made. const auto remaining_physical_space = total_size - total_direct_size; fmem_map.clear(); fmem_map.emplace(total_direct_size, - FlexibleMemoryArea{total_direct_size, remaining_physical_space}); + PhysicalMemoryArea{total_direct_size, remaining_physical_space}); LOG_INFO(Kernel_Vmm, "Configured memory regions: flexible size = {:#x}, direct size = {:#x}", total_flexible_size, total_direct_size); @@ -115,6 +119,7 @@ void MemoryManager::SetPrtArea(u32 id, VAddr address, u64 size) { void MemoryManager::CopySparseMemory(VAddr virtual_addr, u8* dest, u64 size) { ASSERT_MSG(IsValidMapping(virtual_addr), "Attempted to access invalid address {:#x}", virtual_addr); + mutex.lock_shared(); auto vma = FindVMA(virtual_addr); while (size) { @@ -129,23 +134,46 @@ void MemoryManager::CopySparseMemory(VAddr virtual_addr, u8* dest, u64 size) { dest += copy_size; ++vma; } + + mutex.unlock_shared(); } -bool MemoryManager::TryWriteBacking(void* address, const void* data, u32 num_bytes) { +bool MemoryManager::TryWriteBacking(void* address, const void* data, u64 size) { const VAddr virtual_addr = std::bit_cast(address); - ASSERT_MSG(IsValidMapping(virtual_addr, num_bytes), "Attempted to access invalid address {:#x}", + ASSERT_MSG(IsValidMapping(virtual_addr, size), "Attempted to access invalid address {:#x}", virtual_addr); - const auto& vma = FindVMA(virtual_addr)->second; - if (!HasPhysicalBacking(vma)) { - return false; + mutex.lock_shared(); + + std::vector vmas_to_write; + auto current_vma = FindVMA(virtual_addr); + while (virtual_addr + size < current_vma->second.base + current_vma->second.size) { + if (!HasPhysicalBacking(current_vma->second)) { + mutex.unlock_shared(); + return false; + } + vmas_to_write.emplace_back(current_vma->second); + current_vma++; } - u8* backing = impl.BackingBase() + vma.phys_base + (virtual_addr - vma.base); - memcpy(backing, data, num_bytes); + + for (auto& vma : vmas_to_write) { + auto start_in_vma = std::max(virtual_addr, vma.base) - vma.base; + for (auto& phys_area : vma.phys_areas) { + if (!size) { + break; + } + u8* backing = impl.BackingBase() + phys_area.second.base + start_in_vma; + u64 copy_size = std::min(size, phys_area.second.size); + memcpy(backing, data, copy_size); + size -= copy_size; + } + } + + mutex.unlock_shared(); return true; } PAddr MemoryManager::PoolExpand(PAddr search_start, PAddr search_end, u64 size, u64 alignment) { - std::scoped_lock lk{mutex}; + mutex.lock(); alignment = alignment > 0 ? alignment : 64_KB; auto dmem_area = FindDmemArea(search_start); @@ -155,7 +183,7 @@ PAddr MemoryManager::PoolExpand(PAddr search_start, PAddr search_end, u64 size, auto mapping_end = mapping_start + size; // Find the first free, large enough dmem area in the range. - while (dmem_area->second.dma_type != DMAType::Free || + while (dmem_area->second.dma_type != PhysicalMemoryType::Free || dmem_area->second.GetEnd() < mapping_end) { // The current dmem_area isn't suitable, move to the next one. dmem_area++; @@ -171,33 +199,34 @@ PAddr MemoryManager::PoolExpand(PAddr search_start, PAddr search_end, u64 size, if (dmem_area == dmem_map.end()) { // There are no suitable mappings in this range LOG_ERROR(Kernel_Vmm, "Unable to find free direct memory area: size = {:#x}", size); + mutex.unlock(); return -1; } // Add the allocated region to the list and commit its pages. - auto& area = CarveDmemArea(mapping_start, size)->second; - area.dma_type = DMAType::Pooled; + auto& area = CarvePhysArea(dmem_map, mapping_start, size)->second; + area.dma_type = PhysicalMemoryType::Pooled; area.memory_type = 3; // Track how much dmem was allocated for pools. pool_budget += size; + mutex.unlock(); return mapping_start; } PAddr MemoryManager::Allocate(PAddr search_start, PAddr search_end, u64 size, u64 alignment, s32 memory_type) { - std::scoped_lock lk{mutex}; + mutex.lock(); alignment = alignment > 0 ? alignment : 16_KB; auto dmem_area = FindDmemArea(search_start); - auto mapping_start = search_start > dmem_area->second.base - ? Common::AlignUp(search_start, alignment) - : Common::AlignUp(dmem_area->second.base, alignment); + auto mapping_start = + Common::AlignUp(std::max(search_start, dmem_area->second.base), alignment); auto mapping_end = mapping_start + size; // Find the first free, large enough dmem area in the range. - while (dmem_area->second.dma_type != DMAType::Free || + while (dmem_area->second.dma_type != PhysicalMemoryType::Free || dmem_area->second.GetEnd() < mapping_end) { // The current dmem_area isn't suitable, move to the next one. dmem_area++; @@ -213,19 +242,22 @@ PAddr MemoryManager::Allocate(PAddr search_start, PAddr search_end, u64 size, u6 if (dmem_area == dmem_map.end()) { // There are no suitable mappings in this range LOG_ERROR(Kernel_Vmm, "Unable to find free direct memory area: size = {:#x}", size); + mutex.unlock(); return -1; } // Add the allocated region to the list and commit its pages. - auto& area = CarveDmemArea(mapping_start, size)->second; + auto& area = CarvePhysArea(dmem_map, mapping_start, size)->second; area.memory_type = memory_type; - area.dma_type = DMAType::Allocated; + area.dma_type = PhysicalMemoryType::Allocated; MergeAdjacent(dmem_map, dmem_area); + + mutex.unlock(); return mapping_start; } void MemoryManager::Free(PAddr phys_addr, u64 size) { - std::scoped_lock lk{mutex}; + mutex.lock(); // Release any dmem mappings that reference this physical block. std::vector> remove_list; @@ -233,19 +265,21 @@ void MemoryManager::Free(PAddr phys_addr, u64 size) { if (mapping.type != VMAType::Direct) { continue; } - if (mapping.phys_base <= phys_addr && phys_addr < mapping.phys_base + mapping.size) { - const auto vma_start_offset = phys_addr - mapping.phys_base; - const auto addr_in_vma = mapping.base + vma_start_offset; - const auto size_in_vma = - mapping.size - vma_start_offset > size ? size : mapping.size - vma_start_offset; + for (auto& [offset_in_vma, phys_mapping] : mapping.phys_areas) { + if (phys_addr + size > phys_mapping.base && + phys_addr < phys_mapping.base + phys_mapping.size) { + const u64 phys_offset = + std::max(phys_mapping.base, phys_addr) - phys_mapping.base; + const VAddr addr_in_vma = mapping.base + offset_in_vma + phys_offset; + const u64 unmap_size = std::min(phys_mapping.size - phys_offset, size); - LOG_INFO(Kernel_Vmm, "Unmaping direct mapping {:#x} with size {:#x}", addr_in_vma, - size_in_vma); - // Unmaping might erase from vma_map. We can't do it here. - remove_list.emplace_back(addr_in_vma, size_in_vma); + // Unmapping might erase from vma_map. We can't do it here. + remove_list.emplace_back(addr_in_vma, unmap_size); + } } } for (const auto& [addr, size] : remove_list) { + LOG_INFO(Kernel_Vmm, "Unmapping direct mapping {:#x} with size {:#x}", addr, size); UnmapMemoryImpl(addr, size); } @@ -255,15 +289,13 @@ void MemoryManager::Free(PAddr phys_addr, u64 size) { auto dmem_area = FindDmemArea(phys_addr); while (dmem_area != dmem_map.end() && remaining_size > 0) { // Carve a free dmem area in place of this one. - const auto start_phys_addr = - phys_addr > dmem_area->second.base ? phys_addr : dmem_area->second.base; + const auto start_phys_addr = std::max(phys_addr, dmem_area->second.base); const auto offset_in_dma = start_phys_addr - dmem_area->second.base; - const auto size_in_dma = dmem_area->second.size - offset_in_dma > remaining_size - ? remaining_size - : dmem_area->second.size - offset_in_dma; - const auto dmem_handle = CarveDmemArea(start_phys_addr, size_in_dma); + const auto size_in_dma = + std::min(dmem_area->second.size - offset_in_dma, remaining_size); + const auto dmem_handle = CarvePhysArea(dmem_map, start_phys_addr, size_in_dma); auto& new_dmem_area = dmem_handle->second; - new_dmem_area.dma_type = DMAType::Free; + new_dmem_area.dma_type = PhysicalMemoryType::Free; new_dmem_area.memory_type = 0; // Merge the new dmem_area with dmem_map @@ -274,12 +306,14 @@ void MemoryManager::Free(PAddr phys_addr, u64 size) { remaining_size -= size_in_dma; dmem_area = FindDmemArea(phys_addr_to_search); } + + mutex.unlock(); } s32 MemoryManager::PoolCommit(VAddr virtual_addr, u64 size, MemoryProt prot, s32 mtype) { ASSERT_MSG(IsValidMapping(virtual_addr, size), "Attempted to access invalid address {:#x}", virtual_addr); - std::scoped_lock lk{mutex}; + mutex.lock(); // Input addresses to PoolCommit are treated as fixed, and have a constant alignment. const u64 alignment = 64_KB; @@ -289,6 +323,7 @@ s32 MemoryManager::PoolCommit(VAddr virtual_addr, u64 size, MemoryProt prot, s32 if (vma.type != VMAType::PoolReserved) { // If we're attempting to commit non-pooled memory, return EINVAL LOG_ERROR(Kernel_Vmm, "Attempting to commit non-pooled memory at {:#x}", mapped_addr); + mutex.unlock(); return ORBIS_KERNEL_ERROR_EINVAL; } @@ -297,12 +332,14 @@ s32 MemoryManager::PoolCommit(VAddr virtual_addr, u64 size, MemoryProt prot, s32 LOG_ERROR(Kernel_Vmm, "Pooled region {:#x} to {:#x} is not large enough to commit from {:#x} to {:#x}", vma.base, vma.base + vma.size, mapped_addr, mapped_addr + size); + mutex.unlock(); return ORBIS_KERNEL_ERROR_EINVAL; } if (pool_budget <= size) { // If there isn't enough pooled memory to perform the mapping, return ENOMEM LOG_ERROR(Kernel_Vmm, "Not enough pooled memory to perform mapping"); + mutex.unlock(); return ORBIS_KERNEL_ERROR_ENOMEM; } else { // Track how much pooled memory this commit will take @@ -314,48 +351,54 @@ s32 MemoryManager::PoolCommit(VAddr virtual_addr, u64 size, MemoryProt prot, s32 prot |= MemoryProt::CpuRead; } + // Create the virtual mapping for the commit + const auto new_vma_handle = CarveVMA(virtual_addr, size); + auto& new_vma = new_vma_handle->second; + new_vma.disallow_merge = false; + new_vma.prot = prot; + new_vma.name = "anon"; + new_vma.type = Core::VMAType::Pooled; + new_vma.phys_areas.clear(); + // Find suitable physical addresses auto handle = dmem_map.begin(); u64 remaining_size = size; VAddr current_addr = mapped_addr; - while (handle != dmem_map.end() && remaining_size != 0) { - if (handle->second.dma_type != DMAType::Pooled) { + while (handle != dmem_map.end() && remaining_size > 0) { + if (handle->second.dma_type != PhysicalMemoryType::Pooled) { // Non-pooled means it's either not for pool use, or already committed. handle++; continue; } // On PS4, commits can make sparse physical mappings. - // For now, it's easier to create separate memory mappings for each physical mapping. u64 size_to_map = std::min(remaining_size, handle->second.size); - // Carve out the new VMA representing this mapping - const auto new_vma_handle = CarveVMA(current_addr, size_to_map); - auto& new_vma = new_vma_handle->second; - new_vma.disallow_merge = false; - new_vma.prot = prot; - new_vma.name = "anon"; - new_vma.type = Core::VMAType::Pooled; - new_vma.is_exec = false; - // Use the start of this area as the physical backing for this mapping. - const auto new_dmem_handle = CarveDmemArea(handle->second.base, size_to_map); + const auto new_dmem_handle = CarvePhysArea(dmem_map, handle->second.base, size_to_map); auto& new_dmem_area = new_dmem_handle->second; - new_dmem_area.dma_type = DMAType::Committed; + new_dmem_area.dma_type = PhysicalMemoryType::Committed; new_dmem_area.memory_type = mtype; - new_vma.phys_base = new_dmem_area.base; - handle = MergeAdjacent(dmem_map, new_dmem_handle); - // Perform the mapping - void* out_addr = impl.Map(current_addr, size_to_map, alignment, new_vma.phys_base, false); + // Add the dmem area to this vma, merge it with any similar tracked areas. + new_vma.phys_areas[current_addr - mapped_addr] = new_dmem_handle->second; + MergeAdjacent(new_vma.phys_areas, new_vma.phys_areas.find(current_addr - mapped_addr)); + + // Perform an address space mapping for each physical area + void* out_addr = impl.Map(current_addr, size_to_map, new_dmem_area.base); TRACK_ALLOC(out_addr, size_to_map, "VMEM"); + handle = MergeAdjacent(dmem_map, new_dmem_handle); current_addr += size_to_map; remaining_size -= size_to_map; handle++; } - ASSERT_MSG(remaining_size == 0, "Unable to map physical memory"); + ASSERT_MSG(remaining_size == 0, "Failed to commit pooled memory"); + // Merge this VMA with similar nearby areas + MergeAdjacent(vma_map, new_vma_handle); + + mutex.unlock(); if (IsValidGpuMapping(mapped_addr, size)) { rasterizer->MapMemory(mapped_addr, size); } @@ -376,13 +419,15 @@ s32 MemoryManager::MapMemory(void** out_addr, VAddr virtual_addr, u64 size, Memo return ORBIS_KERNEL_ERROR_EINVAL; } - std::scoped_lock lk{mutex}; + mutex.lock(); + PhysHandle dmem_area; // Validate the requested physical address range if (phys_addr != -1) { if (total_direct_size < phys_addr + size) { LOG_ERROR(Kernel_Vmm, "Unable to map {:#x} bytes at physical address {:#x}", size, phys_addr); + mutex.unlock(); return ORBIS_KERNEL_ERROR_ENOMEM; } @@ -390,49 +435,24 @@ s32 MemoryManager::MapMemory(void** out_addr, VAddr virtual_addr, u64 size, Memo auto dmem_area = FindDmemArea(phys_addr); while (dmem_area != dmem_map.end() && dmem_area->second.base < phys_addr + size) { // If any requested dmem area is not allocated, return an error. - if (dmem_area->second.dma_type != DMAType::Allocated && - dmem_area->second.dma_type != DMAType::Mapped) { + if (dmem_area->second.dma_type != PhysicalMemoryType::Allocated && + dmem_area->second.dma_type != PhysicalMemoryType::Mapped) { LOG_ERROR(Kernel_Vmm, "Unable to map {:#x} bytes at physical address {:#x}", size, phys_addr); + mutex.unlock(); return ORBIS_KERNEL_ERROR_ENOMEM; } // If we need to perform extra validation, then check for Mapped dmem areas too. - if (validate_dmem && dmem_area->second.dma_type == DMAType::Mapped) { + if (validate_dmem && dmem_area->second.dma_type == PhysicalMemoryType::Mapped) { LOG_ERROR(Kernel_Vmm, "Unable to map {:#x} bytes at physical address {:#x}", size, phys_addr); + mutex.unlock(); return ORBIS_KERNEL_ERROR_EBUSY; } dmem_area++; } - - // If the prior loop succeeds, we need to loop through again and carve out mapped dmas. - // This needs to be a separate loop to avoid modifying dmem map during failed calls. - auto phys_addr_to_search = phys_addr; - auto remaining_size = size; - dmem_area = FindDmemArea(phys_addr); - while (dmem_area != dmem_map.end() && remaining_size > 0) { - // Carve a new dmem area in place of this one with the appropriate type. - // Ensure the carved area only covers the current dmem area. - const auto start_phys_addr = - phys_addr > dmem_area->second.base ? phys_addr : dmem_area->second.base; - const auto offset_in_dma = start_phys_addr - dmem_area->second.base; - const auto size_in_dma = dmem_area->second.size - offset_in_dma > remaining_size - ? remaining_size - : dmem_area->second.size - offset_in_dma; - const auto dmem_handle = CarveDmemArea(start_phys_addr, size_in_dma); - auto& new_dmem_area = dmem_handle->second; - new_dmem_area.dma_type = DMAType::Mapped; - - // Merge the new dmem_area with dmem_map - MergeAdjacent(dmem_map, dmem_handle); - - // Get the next relevant dmem area. - phys_addr_to_search = phys_addr + size_in_dma; - remaining_size -= size_in_dma; - dmem_area = FindDmemArea(phys_addr_to_search); - } } // Limit the minimum address to SystemManagedVirtualBase to prevent hardware-specific issues. @@ -463,6 +483,7 @@ s32 MemoryManager::MapMemory(void** out_addr, VAddr virtual_addr, u64 size, Memo auto remaining_size = vma.base + vma.size - mapped_addr; if (!vma.IsFree() || remaining_size < size) { LOG_ERROR(Kernel_Vmm, "Unable to map {:#x} bytes at address {:#x}", size, mapped_addr); + mutex.unlock(); return ORBIS_KERNEL_ERROR_ENOMEM; } } else { @@ -473,6 +494,7 @@ s32 MemoryManager::MapMemory(void** out_addr, VAddr virtual_addr, u64 size, Memo mapped_addr = SearchFree(mapped_addr, size, alignment); if (mapped_addr == -1) { // No suitable memory areas to map to + mutex.unlock(); return ORBIS_KERNEL_ERROR_ENOMEM; } } @@ -480,62 +502,109 @@ s32 MemoryManager::MapMemory(void** out_addr, VAddr virtual_addr, u64 size, Memo // Create a memory area representing this mapping. const auto new_vma_handle = CarveVMA(mapped_addr, size); auto& new_vma = new_vma_handle->second; - - // If type is Flexible, we need to track how much flexible memory is used here. - // We also need to determine a reasonable physical base to perform this mapping at. - if (type == VMAType::Flexible) { - flexible_usage += size; - - // Find a suitable physical address - auto handle = fmem_map.begin(); - while (handle != fmem_map.end() && - (!handle->second.is_free || handle->second.size < size)) { - handle++; - } - - // Some games will end up fragmenting the flexible address space. - ASSERT_MSG(handle != fmem_map.end() && handle->second.is_free, - "No suitable physical memory areas to map"); - - // We'll use the start of this area as the physical backing for this mapping. - const auto new_fmem_handle = CarveFmemArea(handle->second.base, size); - auto& new_fmem_area = new_fmem_handle->second; - new_fmem_area.is_free = false; - phys_addr = new_fmem_area.base; - MergeAdjacent(fmem_map, new_fmem_handle); - } - + const bool is_exec = True(prot & MemoryProt::CpuExec); if (True(prot & MemoryProt::CpuWrite)) { // On PS4, read is appended to write mappings. prot |= MemoryProt::CpuRead; } - const bool is_exec = True(prot & MemoryProt::CpuExec); new_vma.disallow_merge = True(flags & MemoryMapFlags::NoCoalesce); new_vma.prot = prot; new_vma.name = name; new_vma.type = type; - new_vma.phys_base = phys_addr == -1 ? 0 : phys_addr; - new_vma.is_exec = is_exec; + new_vma.phys_areas.clear(); - if (type == VMAType::Reserved) { - // Technically this should be done for direct and flexible mappings too, - // But some Windows-specific limitations make that hard to accomplish. + // If type is Flexible, we need to track how much flexible memory is used here. + // We also need to determine a reasonable physical base to perform this mapping at. + if (type == VMAType::Flexible) { + // Find suitable physical addresses + auto handle = fmem_map.begin(); + u64 remaining_size = size; + VAddr current_addr = mapped_addr; + while (handle != fmem_map.end() && remaining_size != 0) { + if (handle->second.dma_type != PhysicalMemoryType::Free) { + // If the handle isn't free, we cannot use it. + handle++; + continue; + } + + // Determine the size we can map here. + u64 size_to_map = std::min(remaining_size, handle->second.size); + + // Create a physical area + const auto new_fmem_handle = CarvePhysArea(fmem_map, handle->second.base, size_to_map); + auto& new_fmem_area = new_fmem_handle->second; + new_fmem_area.dma_type = PhysicalMemoryType::Flexible; + + // Add the new area to the vma, merge it with any similar tracked areas. + new_vma.phys_areas[current_addr - mapped_addr] = new_fmem_handle->second; + MergeAdjacent(new_vma.phys_areas, new_vma.phys_areas.find(current_addr - mapped_addr)); + + // Perform an address space mapping for each physical area + void* out_addr = impl.Map(current_addr, size_to_map, new_fmem_area.base, is_exec); + TRACK_ALLOC(out_addr, size_to_map, "VMEM"); + + handle = MergeAdjacent(fmem_map, new_fmem_handle); + current_addr += size_to_map; + remaining_size -= size_to_map; + flexible_usage += size_to_map; + handle++; + } + ASSERT_MSG(remaining_size == 0, "Failed to map physical memory"); + } else if (type == VMAType::Direct) { + // Map the physical memory for this direct memory mapping. + auto phys_addr_to_search = phys_addr; + u64 remaining_size = size; + dmem_area = FindDmemArea(phys_addr); + while (dmem_area != dmem_map.end() && remaining_size > 0) { + // Carve a new dmem area in place of this one with the appropriate type. + // Ensure the carved area only covers the current dmem area. + const auto start_phys_addr = std::max(phys_addr, dmem_area->second.base); + const auto offset_in_dma = start_phys_addr - dmem_area->second.base; + const auto size_in_dma = + std::min(dmem_area->second.size - offset_in_dma, remaining_size); + const auto dmem_handle = CarvePhysArea(dmem_map, start_phys_addr, size_in_dma); + auto& new_dmem_area = dmem_handle->second; + new_dmem_area.dma_type = PhysicalMemoryType::Mapped; + + // Add the dmem area to this vma, merge it with any similar tracked areas. + new_vma.phys_areas[phys_addr_to_search - phys_addr] = dmem_handle->second; + MergeAdjacent(new_vma.phys_areas, + new_vma.phys_areas.find(phys_addr_to_search - phys_addr)); + + // Merge the new dmem_area with dmem_map + MergeAdjacent(dmem_map, dmem_handle); + + // Get the next relevant dmem area. + phys_addr_to_search = phys_addr + size_in_dma; + remaining_size -= size_in_dma; + dmem_area = FindDmemArea(phys_addr_to_search); + } + ASSERT_MSG(remaining_size == 0, "Failed to map physical memory"); + } + + if (new_vma.type != VMAType::Direct || sdk_version >= Common::ElfInfo::FW_20) { + // Merge this VMA with similar nearby areas + // Direct memory mappings only coalesce on SDK version 2.00 or later. MergeAdjacent(vma_map, new_vma_handle); } - if (type == VMAType::Reserved || type == VMAType::PoolReserved) { - // For Reserved/PoolReserved mappings, we don't perform any address space allocations. - // Just set out_addr to mapped_addr instead. - *out_addr = std::bit_cast(mapped_addr); - } else { + *out_addr = std::bit_cast(mapped_addr); + if (type != VMAType::Reserved && type != VMAType::PoolReserved) { + // Flexible address space mappings were performed while finding direct memory areas. + if (type != VMAType::Flexible) { + impl.Map(mapped_addr, size, phys_addr, is_exec); + } + TRACK_ALLOC(*out_addr, size, "VMEM"); + + mutex.unlock(); + // If this is not a reservation, then map to GPU and address space if (IsValidGpuMapping(mapped_addr, size)) { rasterizer->MapMemory(mapped_addr, size); } - *out_addr = impl.Map(mapped_addr, size, alignment, phys_addr, is_exec); - - TRACK_ALLOC(*out_addr, size, "VMEM"); + } else { + mutex.unlock(); } return ORBIS_OK; @@ -547,13 +616,14 @@ s32 MemoryManager::MapFile(void** out_addr, VAddr virtual_addr, u64 size, Memory ASSERT_MSG(IsValidMapping(mapped_addr, size), "Attempted to access invalid address {:#x}", mapped_addr); - std::scoped_lock lk{mutex}; + mutex.lock(); // Find first free area to map the file. if (False(flags & MemoryMapFlags::Fixed)) { mapped_addr = SearchFree(mapped_addr, size, 1); if (mapped_addr == -1) { // No suitable memory areas to map to + mutex.unlock(); return ORBIS_KERNEL_ERROR_ENOMEM; } } @@ -571,11 +641,13 @@ s32 MemoryManager::MapFile(void** out_addr, VAddr virtual_addr, u64 size, Memory auto file = h->GetFile(fd); if (file == nullptr) { LOG_WARNING(Kernel_Vmm, "Invalid file for mmap, fd {}", fd); + mutex.unlock(); return ORBIS_KERNEL_ERROR_EBADF; } if (file->type != Core::FileSys::FileType::Regular) { LOG_WARNING(Kernel_Vmm, "Unsupported file type for mmap, fd {}", fd); + mutex.unlock(); return ORBIS_KERNEL_ERROR_EBADF; } @@ -612,6 +684,8 @@ s32 MemoryManager::MapFile(void** out_addr, VAddr virtual_addr, u64 size, Memory new_vma.fd = fd; new_vma.type = VMAType::File; + mutex.unlock(); + *out_addr = std::bit_cast(mapped_addr); return ORBIS_OK; } @@ -619,13 +693,14 @@ s32 MemoryManager::MapFile(void** out_addr, VAddr virtual_addr, u64 size, Memory s32 MemoryManager::PoolDecommit(VAddr virtual_addr, u64 size) { ASSERT_MSG(IsValidMapping(virtual_addr, size), "Attempted to access invalid address {:#x}", virtual_addr); - std::scoped_lock lk{mutex}; + mutex.lock(); // Do an initial search to ensure this decommit is valid. auto it = FindVMA(virtual_addr); while (it != vma_map.end() && it->second.base + it->second.size <= virtual_addr + size) { if (it->second.type != VMAType::PoolReserved && it->second.type != VMAType::Pooled) { LOG_ERROR(Kernel_Vmm, "Attempting to decommit non-pooled memory!"); + mutex.unlock(); return ORBIS_KERNEL_ERROR_EINVAL; } it++; @@ -635,36 +710,46 @@ s32 MemoryManager::PoolDecommit(VAddr virtual_addr, u64 size) { u64 remaining_size = size; VAddr current_addr = virtual_addr; while (remaining_size != 0) { - const auto it = FindVMA(current_addr); - const auto& vma_base = it->second; - const bool is_exec = vma_base.is_exec; + const auto handle = FindVMA(current_addr); + const auto& vma_base = handle->second; const auto start_in_vma = current_addr - vma_base.base; const auto size_in_vma = std::min(remaining_size, vma_base.size - start_in_vma); if (vma_base.type == VMAType::Pooled) { // We always map PoolCommitted memory to GPU, so unmap when decomitting. if (IsValidGpuMapping(current_addr, size_in_vma)) { + mutex.unlock(); rasterizer->UnmapMemory(current_addr, size_in_vma); + mutex.lock(); } // Track how much pooled memory is decommitted pool_budget += size_in_vma; // Re-pool the direct memory used by this mapping - const auto unmap_phys_base = vma_base.phys_base + start_in_vma; - const auto new_dmem_handle = CarveDmemArea(unmap_phys_base, size_in_vma); - auto& new_dmem_area = new_dmem_handle->second; - new_dmem_area.dma_type = DMAType::Pooled; + u64 size_to_free = size_in_vma; + auto phys_handle = std::prev(vma_base.phys_areas.upper_bound(start_in_vma)); + while (phys_handle != vma_base.phys_areas.end() && size_to_free > 0) { + // Calculate physical memory offset, address, and size + u64 dma_offset = + std::max(phys_handle->first, start_in_vma) - phys_handle->first; + PAddr phys_addr = phys_handle->second.base + dma_offset; + u64 size_in_dma = + std::min(size_to_free, phys_handle->second.size - dma_offset); - // Coalesce with nearby direct memory areas. - MergeAdjacent(dmem_map, new_dmem_handle); - } + // Create a new dmem area reflecting the pooled region + const auto new_dmem_handle = CarvePhysArea(dmem_map, phys_addr, size_in_dma); + auto& new_dmem_area = new_dmem_handle->second; + new_dmem_area.dma_type = PhysicalMemoryType::Pooled; - if (vma_base.type != VMAType::PoolReserved) { - // Unmap the memory region. - impl.Unmap(vma_base.base, vma_base.size, start_in_vma, start_in_vma + size_in_vma, - vma_base.phys_base, vma_base.is_exec, true, false); - TRACK_FREE(virtual_addr, "VMEM"); + // Coalesce with nearby direct memory areas. + MergeAdjacent(dmem_map, new_dmem_handle); + + // Increment loop + size_to_free -= size_in_dma; + phys_handle++; + } + ASSERT_MSG(size_to_free == 0, "Failed to decommit pooled memory"); } // Mark region as pool reserved and attempt to coalesce it with neighbours. @@ -672,108 +757,116 @@ s32 MemoryManager::PoolDecommit(VAddr virtual_addr, u64 size) { auto& vma = new_it->second; vma.type = VMAType::PoolReserved; vma.prot = MemoryProt::NoAccess; - vma.phys_base = 0; vma.disallow_merge = false; vma.name = "anon"; + vma.phys_areas.clear(); MergeAdjacent(vma_map, new_it); current_addr += size_in_vma; remaining_size -= size_in_vma; } + // Unmap from address space + impl.Unmap(virtual_addr, size, true); + TRACK_FREE(virtual_addr, "VMEM"); + + mutex.unlock(); return ORBIS_OK; } s32 MemoryManager::UnmapMemory(VAddr virtual_addr, u64 size) { - std::scoped_lock lk{mutex}; if (size == 0) { return ORBIS_OK; } + mutex.lock(); virtual_addr = Common::AlignDown(virtual_addr, 16_KB); size = Common::AlignUp(size, 16_KB); ASSERT_MSG(IsValidMapping(virtual_addr, size), "Attempted to access invalid address {:#x}", virtual_addr); - return UnmapMemoryImpl(virtual_addr, size); + u64 bytes_unmapped = UnmapMemoryImpl(virtual_addr, size); + mutex.unlock(); + return bytes_unmapped; } u64 MemoryManager::UnmapBytesFromEntry(VAddr virtual_addr, VirtualMemoryArea vma_base, u64 size) { - const auto vma_base_addr = vma_base.base; - const auto vma_base_size = vma_base.size; - const auto type = vma_base.type; - const auto phys_base = vma_base.phys_base; - const bool is_exec = vma_base.is_exec; - const auto start_in_vma = virtual_addr - vma_base_addr; - const auto adjusted_size = - vma_base_size - start_in_vma < size ? vma_base_size - start_in_vma : size; - const bool has_backing = HasPhysicalBacking(vma_base) || type == VMAType::File; - const auto prot = vma_base.prot; - const bool readonly_file = prot == MemoryProt::CpuRead && type == VMAType::File; + const auto start_in_vma = virtual_addr - vma_base.base; + const auto size_in_vma = std::min(vma_base.size - start_in_vma, size); + const auto vma_type = vma_base.type; + const bool has_backing = HasPhysicalBacking(vma_base) || vma_base.type == VMAType::File; + const bool readonly_file = + vma_base.prot == MemoryProt::CpuRead && vma_base.type == VMAType::File; + const bool is_exec = True(vma_base.prot & MemoryProt::CpuExec); - if (type == VMAType::Free) { - return adjusted_size; + if (vma_base.type == VMAType::Free || vma_base.type == VMAType::Pooled) { + return size_in_vma; } - if (type == VMAType::Direct) { - // Unmap all direct memory areas covered by this unmap. - auto phys_addr = phys_base + start_in_vma; - auto remaining_size = adjusted_size; - DMemHandle dmem_handle = FindDmemArea(phys_addr); - while (dmem_handle != dmem_map.end() && remaining_size > 0) { - const auto start_in_dma = phys_addr - dmem_handle->second.base; - const auto size_in_dma = dmem_handle->second.size - start_in_dma > remaining_size - ? remaining_size - : dmem_handle->second.size - start_in_dma; - dmem_handle = CarveDmemArea(phys_addr, size_in_dma); - auto& dmem_area = dmem_handle->second; - dmem_area.dma_type = DMAType::Allocated; - remaining_size -= dmem_area.size; - phys_addr += dmem_area.size; + PAddr phys_base = 0; + VAddr current_addr = virtual_addr; + if (vma_base.phys_areas.size() > 0) { + u64 size_to_free = size_in_vma; + auto phys_handle = std::prev(vma_base.phys_areas.upper_bound(start_in_vma)); + while (phys_handle != vma_base.phys_areas.end() && size_to_free > 0) { + // Calculate physical memory offset, address, and size + u64 dma_offset = std::max(phys_handle->first, start_in_vma) - phys_handle->first; + PAddr phys_addr = phys_handle->second.base + dma_offset; + u64 size_in_dma = std::min(size_to_free, phys_handle->second.size - dma_offset); - // Check if we can coalesce any dmem areas. - MergeAdjacent(dmem_map, dmem_handle); - dmem_handle = FindDmemArea(phys_addr); + // Create a new dmem area reflecting the pooled region + if (vma_type == VMAType::Direct) { + const auto new_dmem_handle = CarvePhysArea(dmem_map, phys_addr, size_in_dma); + auto& new_dmem_area = new_dmem_handle->second; + new_dmem_area.dma_type = PhysicalMemoryType::Allocated; + + // Coalesce with nearby direct memory areas. + MergeAdjacent(dmem_map, new_dmem_handle); + } else if (vma_type == VMAType::Flexible) { + // Update fmem_map + const auto new_fmem_handle = CarvePhysArea(fmem_map, phys_addr, size_in_dma); + auto& new_fmem_area = new_fmem_handle->second; + new_fmem_area.dma_type = PhysicalMemoryType::Free; + + // Coalesce with nearby flexible memory areas. + MergeAdjacent(fmem_map, new_fmem_handle); + + // Zero out the old memory data + const auto unmap_hardware_address = impl.BackingBase() + phys_addr; + std::memset(unmap_hardware_address, 0, size_in_dma); + + // Update flexible usage + flexible_usage -= size_in_dma; + } + + // Increment through loop + size_to_free -= size_in_dma; + phys_handle++; } - } - - if (type == VMAType::Flexible) { - flexible_usage -= adjusted_size; - - // Now that there is a physical backing used for flexible memory, - // manually erase the contents before unmapping to prevent possible issues. - const auto unmap_hardware_address = impl.BackingBase() + phys_base + start_in_vma; - std::memset(unmap_hardware_address, 0, adjusted_size); - - // Address space unmap needs the physical_base from the start of the vma, - // so calculate the phys_base to unmap from here. - const auto unmap_phys_base = phys_base + start_in_vma; - const auto new_fmem_handle = CarveFmemArea(unmap_phys_base, adjusted_size); - auto& new_fmem_area = new_fmem_handle->second; - new_fmem_area.is_free = true; - MergeAdjacent(fmem_map, new_fmem_handle); + ASSERT_MSG(size_to_free == 0, "Failed to unmap physical memory"); } // Mark region as free and attempt to coalesce it with neighbours. - const auto new_it = CarveVMA(virtual_addr, adjusted_size); + const auto new_it = CarveVMA(virtual_addr, size_in_vma); auto& vma = new_it->second; vma.type = VMAType::Free; vma.prot = MemoryProt::NoAccess; - vma.phys_base = 0; + vma.phys_areas.clear(); vma.disallow_merge = false; vma.name = ""; MergeAdjacent(vma_map, new_it); - if (type != VMAType::Reserved && type != VMAType::PoolReserved) { + if (vma_type != VMAType::Reserved && vma_type != VMAType::PoolReserved) { + // Unmap the memory region. + impl.Unmap(virtual_addr, size_in_vma, has_backing); + TRACK_FREE(virtual_addr, "VMEM"); + // If this mapping has GPU access, unmap from GPU. if (IsValidGpuMapping(virtual_addr, size)) { + mutex.unlock(); rasterizer->UnmapMemory(virtual_addr, size); + mutex.lock(); } - - // Unmap the memory region. - impl.Unmap(vma_base_addr, vma_base_size, start_in_vma, start_in_vma + adjusted_size, - phys_base, is_exec, has_backing, readonly_file); - TRACK_FREE(virtual_addr, "VMEM"); } - return adjusted_size; + return size_in_vma; } s32 MemoryManager::UnmapMemoryImpl(VAddr virtual_addr, u64 size) { @@ -792,12 +885,13 @@ s32 MemoryManager::UnmapMemoryImpl(VAddr virtual_addr, u64 size) { s32 MemoryManager::QueryProtection(VAddr addr, void** start, void** end, u32* prot) { ASSERT_MSG(IsValidMapping(addr), "Attempted to access invalid address {:#x}", addr); - std::scoped_lock lk{mutex}; + mutex.lock_shared(); const auto it = FindVMA(addr); const auto& vma = it->second; if (vma.IsFree()) { LOG_ERROR(Kernel_Vmm, "Address {:#x} is not mapped", addr); + mutex.unlock_shared(); return ORBIS_KERNEL_ERROR_EACCES; } @@ -810,6 +904,8 @@ s32 MemoryManager::QueryProtection(VAddr addr, void** start, void** end, u32* pr if (prot != nullptr) { *prot = static_cast(vma.prot); } + + mutex.unlock_shared(); return ORBIS_OK; } @@ -872,8 +968,6 @@ s64 MemoryManager::ProtectBytes(VAddr addr, VirtualMemoryArea& vma_base, u64 siz } s32 MemoryManager::Protect(VAddr addr, u64 size, MemoryProt prot) { - std::scoped_lock lk{mutex}; - // If size is zero, then there's nothing to protect if (size == 0) { return ORBIS_OK; @@ -887,6 +981,7 @@ s32 MemoryManager::Protect(VAddr addr, u64 size, MemoryProt prot) { MemoryProt::CpuReadWrite | MemoryProt::CpuExec | MemoryProt::GpuReadWrite; MemoryProt valid_flags = prot & flag_mask; + mutex.lock(); // Protect all VMAs between addr and addr + size. s64 protected_bytes = 0; while (protected_bytes < size) { @@ -899,18 +994,18 @@ s32 MemoryManager::Protect(VAddr addr, u64 size, MemoryProt prot) { auto result = ProtectBytes(addr + protected_bytes, vma_base, size - protected_bytes, prot); if (result < 0) { // ProtectBytes returned an error, return it + mutex.unlock(); return result; } protected_bytes += result; } + mutex.unlock(); return ORBIS_OK; } s32 MemoryManager::VirtualQuery(VAddr addr, s32 flags, ::Libraries::Kernel::OrbisVirtualQueryInfo* info) { - std::scoped_lock lk{mutex}; - // FindVMA on addresses before the vma_map return garbage data. auto query_addr = addr < impl.SystemManagedVirtualBase() ? impl.SystemManagedVirtualBase() : addr; @@ -918,6 +1013,8 @@ s32 MemoryManager::VirtualQuery(VAddr addr, s32 flags, LOG_WARNING(Kernel_Vmm, "VirtualQuery on free memory region"); return ORBIS_KERNEL_ERROR_EACCES; } + + mutex.lock_shared(); auto it = FindVMA(query_addr); while (it != vma_map.end() && it->second.type == VMAType::Free && flags == 1) { @@ -925,6 +1022,7 @@ s32 MemoryManager::VirtualQuery(VAddr addr, s32 flags, } if (it == vma_map.end() || it->second.type == VMAType::Free) { LOG_WARNING(Kernel_Vmm, "VirtualQuery on free memory region"); + mutex.unlock_shared(); return ORBIS_KERNEL_ERROR_EACCES; } @@ -938,9 +1036,12 @@ s32 MemoryManager::VirtualQuery(VAddr addr, s32 flags, info->is_stack = vma.type == VMAType::Stack ? 1 : 0; info->is_pooled = vma.type == VMAType::PoolReserved || vma.type == VMAType::Pooled ? 1 : 0; info->is_committed = vma.IsMapped() ? 1 : 0; - if (vma.type == VMAType::Direct || vma.type == VMAType::Pooled) { - // Offset is only assigned for direct and pooled mappings. - info->offset = vma.phys_base; + info->memory_type = 0; + if (vma.type == VMAType::Direct) { + // Offset is only assigned for direct mappings. + ASSERT_MSG(vma.phys_areas.size() > 0, "No physical backing for direct mapping?"); + info->offset = vma.phys_areas.begin()->second.base; + info->memory_type = vma.phys_areas.begin()->second.memory_type; } if (vma.type == VMAType::Reserved || vma.type == VMAType::PoolReserved) { // Protection is hidden from reserved mappings. @@ -949,34 +1050,27 @@ s32 MemoryManager::VirtualQuery(VAddr addr, s32 flags, strncpy(info->name, vma.name.data(), ::Libraries::Kernel::ORBIS_KERNEL_MAXIMUM_NAME_LENGTH); - if (vma.type == VMAType::Direct) { - const auto dmem_it = FindDmemArea(vma.phys_base); - ASSERT_MSG(vma.phys_base <= dmem_it->second.GetEnd(), "vma.phys_base is not in dmem_map!"); - info->memory_type = dmem_it->second.memory_type; - } else { - info->memory_type = ::Libraries::Kernel::ORBIS_KERNEL_WB_ONION; - } - + mutex.unlock_shared(); return ORBIS_OK; } s32 MemoryManager::DirectMemoryQuery(PAddr addr, bool find_next, ::Libraries::Kernel::OrbisQueryInfo* out_info) { - std::scoped_lock lk{mutex}; - if (addr >= total_direct_size) { LOG_WARNING(Kernel_Vmm, "Unable to find allocated direct memory region to query!"); return ORBIS_KERNEL_ERROR_EACCES; } + mutex.lock_shared(); auto dmem_area = FindDmemArea(addr); - while (dmem_area != dmem_map.end() && dmem_area->second.dma_type == DMAType::Free && + while (dmem_area != dmem_map.end() && dmem_area->second.dma_type == PhysicalMemoryType::Free && find_next) { dmem_area++; } - if (dmem_area == dmem_map.end() || dmem_area->second.dma_type == DMAType::Free) { + if (dmem_area == dmem_map.end() || dmem_area->second.dma_type == PhysicalMemoryType::Free) { LOG_WARNING(Kernel_Vmm, "Unable to find allocated direct memory region to query!"); + mutex.unlock_shared(); return ORBIS_KERNEL_ERROR_EACCES; } @@ -986,25 +1080,26 @@ s32 MemoryManager::DirectMemoryQuery(PAddr addr, bool find_next, // Loop through all sequential mapped or allocated dmem areas // to determine the hardware accurate end. while (dmem_area != dmem_map.end() && dmem_area->second.memory_type == out_info->memoryType && - (dmem_area->second.dma_type == DMAType::Mapped || - dmem_area->second.dma_type == DMAType::Allocated)) { + (dmem_area->second.dma_type == PhysicalMemoryType::Mapped || + dmem_area->second.dma_type == PhysicalMemoryType::Allocated)) { out_info->end = dmem_area->second.GetEnd(); dmem_area++; } + mutex.unlock_shared(); return ORBIS_OK; } s32 MemoryManager::DirectQueryAvailable(PAddr search_start, PAddr search_end, u64 alignment, PAddr* phys_addr_out, u64* size_out) { - std::scoped_lock lk{mutex}; + mutex.lock_shared(); auto dmem_area = FindDmemArea(search_start); PAddr paddr{}; u64 max_size{}; while (dmem_area != dmem_map.end()) { - if (dmem_area->second.dma_type != DMAType::Free) { + if (dmem_area->second.dma_type != PhysicalMemoryType::Free) { dmem_area++; continue; } @@ -1037,18 +1132,20 @@ s32 MemoryManager::DirectQueryAvailable(PAddr search_start, PAddr search_end, u6 dmem_area++; } + mutex.unlock_shared(); *phys_addr_out = paddr; *size_out = max_size; return ORBIS_OK; } s32 MemoryManager::SetDirectMemoryType(VAddr addr, u64 size, s32 memory_type) { - std::scoped_lock lk{mutex}; + mutex.lock(); ASSERT_MSG(IsValidMapping(addr, size), "Attempted to access invalid address {:#x}", addr); // Search through all VMAs covered by the provided range. // We aren't modifying these VMAs, so it's safe to iterate through them. + VAddr current_addr = addr; auto remaining_size = size; auto vma_handle = FindVMA(addr); while (vma_handle != vma_map.end() && vma_handle->second.base < addr + size) { @@ -1056,40 +1153,42 @@ s32 MemoryManager::SetDirectMemoryType(VAddr addr, u64 size, s32 memory_type) { if (vma_handle->second.type == VMAType::Direct || vma_handle->second.type == VMAType::Pooled) { // Calculate position in vma - const auto start_in_vma = addr - vma_handle->second.base; + const auto start_in_vma = current_addr - vma_handle->second.base; const auto size_in_vma = vma_handle->second.size - start_in_vma; - auto phys_addr = vma_handle->second.phys_base + start_in_vma; - auto size_to_modify = remaining_size > size_in_vma ? size_in_vma : remaining_size; + const auto base_phys_addr = vma_handle->second.phys_areas.begin()->second.base; + auto size_to_modify = std::min(remaining_size, size_in_vma); + for (auto& phys_handle : vma_handle->second.phys_areas) { + if (size_to_modify == 0) { + break; + } - // Loop through remaining dmem areas until the physical addresses represented - // are all adjusted. - DMemHandle dmem_handle = FindDmemArea(phys_addr); - while (dmem_handle != dmem_map.end() && size_in_vma >= size_to_modify && - size_to_modify > 0) { - const auto start_in_dma = phys_addr - dmem_handle->second.base; - const auto size_in_dma = dmem_handle->second.size - start_in_dma > size_to_modify - ? size_to_modify - : dmem_handle->second.size - start_in_dma; - dmem_handle = CarveDmemArea(phys_addr, size_in_dma); + const auto current_phys_addr = + std::max(base_phys_addr, phys_handle.second.base); + if (current_phys_addr >= phys_handle.second.base + phys_handle.second.size) { + continue; + } + const auto start_in_dma = current_phys_addr - phys_handle.second.base; + const auto size_in_dma = phys_handle.second.size - start_in_dma; + + phys_handle.second.memory_type = memory_type; + + auto dmem_handle = CarvePhysArea(dmem_map, current_phys_addr, size_in_dma); auto& dmem_area = dmem_handle->second; dmem_area.memory_type = memory_type; size_to_modify -= dmem_area.size; - phys_addr += dmem_area.size; - - // Check if we can coalesce any dmem areas now that the types are different. MergeAdjacent(dmem_map, dmem_handle); - dmem_handle = FindDmemArea(phys_addr); } } remaining_size -= vma_handle->second.size; vma_handle++; } + mutex.unlock(); return ORBIS_OK; } void MemoryManager::NameVirtualRange(VAddr virtual_addr, u64 size, std::string_view name) { - std::scoped_lock lk{mutex}; + mutex.lock(); // Sizes are aligned up to the nearest 16_KB auto aligned_size = Common::AlignUp(size, 16_KB); @@ -1116,6 +1215,8 @@ void MemoryManager::NameVirtualRange(VAddr virtual_addr, u64 size, std::string_v current_addr += it->second.size; it = FindVMA(current_addr); } + + mutex.unlock(); } s32 MemoryManager::GetDirectMemoryType(PAddr addr, s32* directMemoryTypeOut, @@ -1125,22 +1226,27 @@ s32 MemoryManager::GetDirectMemoryType(PAddr addr, s32* directMemoryTypeOut, return ORBIS_KERNEL_ERROR_ENOENT; } + mutex.lock_shared(); const auto& dmem_area = FindDmemArea(addr)->second; - if (dmem_area.dma_type == DMAType::Free) { + if (dmem_area.dma_type == PhysicalMemoryType::Free) { LOG_ERROR(Kernel_Vmm, "Unable to find allocated direct memory region to check type!"); + mutex.unlock_shared(); return ORBIS_KERNEL_ERROR_ENOENT; } *directMemoryStartOut = reinterpret_cast(dmem_area.base); *directMemoryEndOut = reinterpret_cast(dmem_area.GetEnd()); *directMemoryTypeOut = dmem_area.memory_type; + mutex.unlock_shared(); return ORBIS_OK; } s32 MemoryManager::IsStack(VAddr addr, void** start, void** end) { ASSERT_MSG(IsValidMapping(addr), "Attempted to access invalid address {:#x}", addr); + mutex.lock_shared(); const auto& vma = FindVMA(addr)->second; if (vma.IsFree()) { + mutex.unlock_shared(); return ORBIS_KERNEL_ERROR_EACCES; } @@ -1159,11 +1265,12 @@ s32 MemoryManager::IsStack(VAddr addr, void** start, void** end) { *end = reinterpret_cast(stack_end); } + mutex.unlock_shared(); return ORBIS_OK; } s32 MemoryManager::GetMemoryPoolStats(::Libraries::Kernel::OrbisKernelMemoryPoolBlockStats* stats) { - std::scoped_lock lk{mutex}; + mutex.lock_shared(); // Run through dmem_map, determine how much physical memory is currently committed constexpr u64 block_size = 64_KB; @@ -1171,7 +1278,7 @@ s32 MemoryManager::GetMemoryPoolStats(::Libraries::Kernel::OrbisKernelMemoryPool auto dma_handle = dmem_map.begin(); while (dma_handle != dmem_map.end()) { - if (dma_handle->second.dma_type == DMAType::Committed) { + if (dma_handle->second.dma_type == PhysicalMemoryType::Committed) { committed_size += dma_handle->second.size; } dma_handle++; @@ -1182,6 +1289,8 @@ s32 MemoryManager::GetMemoryPoolStats(::Libraries::Kernel::OrbisKernelMemoryPool // TODO: Determine how "cached blocks" work stats->allocated_cached_blocks = 0; stats->available_cached_blocks = 0; + + mutex.unlock_shared(); return ORBIS_OK; } @@ -1251,6 +1360,52 @@ VAddr MemoryManager::SearchFree(VAddr virtual_addr, u64 size, u32 alignment) { return -1; } +MemoryManager::VMAHandle MemoryManager::MergeAdjacent(VMAMap& handle_map, VMAHandle iter) { + const auto next_vma = std::next(iter); + if (next_vma != handle_map.end() && iter->second.CanMergeWith(next_vma->second)) { + u64 base_offset = iter->second.size; + iter->second.size += next_vma->second.size; + for (auto& area : next_vma->second.phys_areas) { + iter->second.phys_areas[base_offset + area.first] = area.second; + } + handle_map.erase(next_vma); + } + + if (iter != handle_map.begin()) { + auto prev_vma = std::prev(iter); + if (prev_vma->second.CanMergeWith(iter->second)) { + u64 base_offset = prev_vma->second.size; + prev_vma->second.size += iter->second.size; + for (auto& area : iter->second.phys_areas) { + prev_vma->second.phys_areas[base_offset + area.first] = area.second; + } + handle_map.erase(iter); + iter = prev_vma; + } + } + + return iter; +} + +MemoryManager::PhysHandle MemoryManager::MergeAdjacent(PhysMap& handle_map, PhysHandle iter) { + const auto next_vma = std::next(iter); + if (next_vma != handle_map.end() && iter->second.CanMergeWith(next_vma->second)) { + iter->second.size += next_vma->second.size; + handle_map.erase(next_vma); + } + + if (iter != handle_map.begin()) { + auto prev_vma = std::prev(iter); + if (prev_vma->second.CanMergeWith(iter->second)) { + prev_vma->second.size += iter->second.size; + handle_map.erase(iter); + iter = prev_vma; + } + } + + return iter; +} + MemoryManager::VMAHandle MemoryManager::CarveVMA(VAddr virtual_addr, u64 size) { auto vma_handle = FindVMA(virtual_addr); @@ -1279,11 +1434,11 @@ MemoryManager::VMAHandle MemoryManager::CarveVMA(VAddr virtual_addr, u64 size) { return vma_handle; } -MemoryManager::DMemHandle MemoryManager::CarveDmemArea(PAddr addr, u64 size) { - auto dmem_handle = FindDmemArea(addr); - ASSERT_MSG(addr <= dmem_handle->second.GetEnd(), "Physical address not in dmem_map"); +MemoryManager::PhysHandle MemoryManager::CarvePhysArea(PhysMap& map, PAddr addr, u64 size) { + auto pmem_handle = std::prev(map.upper_bound(addr)); + ASSERT_MSG(addr <= pmem_handle->second.GetEnd(), "Physical address not in map"); - const DirectMemoryArea& area = dmem_handle->second; + const PhysicalMemoryArea& area = pmem_handle->second; ASSERT_MSG(area.base <= addr, "Adding an allocation to already allocated region"); const PAddr start_in_area = addr - area.base; @@ -1293,38 +1448,14 @@ MemoryManager::DMemHandle MemoryManager::CarveDmemArea(PAddr addr, u64 size) { if (end_in_vma != area.size) { // Split VMA at the end of the allocated region - Split(dmem_handle, end_in_vma); + Split(map, pmem_handle, end_in_vma); } if (start_in_area != 0) { // Split VMA at the start of the allocated region - dmem_handle = Split(dmem_handle, start_in_area); + pmem_handle = Split(map, pmem_handle, start_in_area); } - return dmem_handle; -} - -MemoryManager::FMemHandle MemoryManager::CarveFmemArea(PAddr addr, u64 size) { - auto fmem_handle = FindFmemArea(addr); - ASSERT_MSG(addr <= fmem_handle->second.GetEnd(), "Physical address not in fmem_map"); - - const FlexibleMemoryArea& area = fmem_handle->second; - ASSERT_MSG(area.base <= addr, "Adding an allocation to already allocated region"); - - const PAddr start_in_area = addr - area.base; - const PAddr end_in_vma = start_in_area + size; - ASSERT_MSG(end_in_vma <= area.size, "Mapping cannot fit inside free region: size = {:#x}", - size); - - if (end_in_vma != area.size) { - // Split VMA at the end of the allocated region - Split(fmem_handle, end_in_vma); - } - if (start_in_area != 0) { - // Split VMA at the start of the allocated region - fmem_handle = Split(fmem_handle, start_in_area); - } - - return fmem_handle; + return pmem_handle; } MemoryManager::VMAHandle MemoryManager::Split(VMAHandle vma_handle, u64 offset_in_vma) { @@ -1337,13 +1468,43 @@ MemoryManager::VMAHandle MemoryManager::Split(VMAHandle vma_handle, u64 offset_i new_vma.size -= offset_in_vma; if (HasPhysicalBacking(new_vma)) { - new_vma.phys_base += offset_in_vma; + // Update physical areas map for both areas + new_vma.phys_areas.clear(); + + std::map old_vma_phys_areas; + for (auto& [offset, region] : old_vma.phys_areas) { + // Fully contained in first VMA + if (offset + region.size <= offset_in_vma) { + old_vma_phys_areas[offset] = region; + } + // Split between both VMAs + if (offset < offset_in_vma && offset + region.size > offset_in_vma) { + // Create region in old VMA + u64 size_in_old = offset_in_vma - offset; + old_vma_phys_areas[offset] = PhysicalMemoryArea{ + region.base, size_in_old, region.memory_type, region.dma_type}; + // Create region in new VMA + PAddr new_base = region.base + size_in_old; + u64 size_in_new = region.size - size_in_old; + new_vma.phys_areas[0] = + PhysicalMemoryArea{new_base, size_in_new, region.memory_type, region.dma_type}; + } + // Fully contained in new VMA + if (offset >= offset_in_vma) { + new_vma.phys_areas[offset - offset_in_vma] = region; + } + } + + // Set old_vma's physical areas map to the newly created map + old_vma.phys_areas = old_vma_phys_areas; } + return vma_map.emplace_hint(std::next(vma_handle), new_vma.base, new_vma); } -MemoryManager::DMemHandle MemoryManager::Split(DMemHandle dmem_handle, u64 offset_in_area) { - auto& old_area = dmem_handle->second; +MemoryManager::PhysHandle MemoryManager::Split(PhysMap& map, PhysHandle phys_handle, + u64 offset_in_area) { + auto& old_area = phys_handle->second; ASSERT(offset_in_area < old_area.size && offset_in_area > 0); auto new_area = old_area; @@ -1352,19 +1513,7 @@ MemoryManager::DMemHandle MemoryManager::Split(DMemHandle dmem_handle, u64 offse new_area.base += offset_in_area; new_area.size -= offset_in_area; - return dmem_map.emplace_hint(std::next(dmem_handle), new_area.base, new_area); -} - -MemoryManager::FMemHandle MemoryManager::Split(FMemHandle fmem_handle, u64 offset_in_area) { - auto& old_area = fmem_handle->second; - ASSERT(offset_in_area < old_area.size && offset_in_area > 0); - - auto new_area = old_area; - old_area.size = offset_in_area; - new_area.base += offset_in_area; - new_area.size -= offset_in_area; - - return fmem_map.emplace_hint(std::next(fmem_handle), new_area.base, new_area); + return map.emplace_hint(std::next(phys_handle), new_area.base, new_area); } } // namespace Core diff --git a/src/core/memory.h b/src/core/memory.h index 7ebf9d34c..0664ed592 100644 --- a/src/core/memory.h +++ b/src/core/memory.h @@ -8,6 +8,7 @@ #include #include #include "common/enum.h" +#include "common/shared_first_mutex.h" #include "common/singleton.h" #include "common/types.h" #include "core/address_space.h" @@ -54,12 +55,37 @@ enum class MemoryMapFlags : u32 { }; DECLARE_ENUM_FLAG_OPERATORS(MemoryMapFlags) -enum class DMAType : u32 { +enum class PhysicalMemoryType : u32 { Free = 0, Allocated = 1, Mapped = 2, Pooled = 3, Committed = 4, + Flexible = 5, +}; + +struct PhysicalMemoryArea { + PAddr base = 0; + u64 size = 0; + s32 memory_type = 0; + PhysicalMemoryType dma_type = PhysicalMemoryType::Free; + + PAddr GetEnd() const { + return base + size; + } + + bool CanMergeWith(const PhysicalMemoryArea& next) const { + if (base + size != next.base) { + return false; + } + if (memory_type != next.memory_type) { + return false; + } + if (dma_type != next.dma_type) { + return false; + } + return true; + } }; enum class VMAType : u32 { @@ -74,60 +100,15 @@ enum class VMAType : u32 { File = 8, }; -struct DirectMemoryArea { - PAddr base = 0; - u64 size = 0; - s32 memory_type = 0; - DMAType dma_type = DMAType::Free; - - PAddr GetEnd() const { - return base + size; - } - - bool CanMergeWith(const DirectMemoryArea& next) const { - if (base + size != next.base) { - return false; - } - if (memory_type != next.memory_type) { - return false; - } - if (dma_type != next.dma_type) { - return false; - } - return true; - } -}; - -struct FlexibleMemoryArea { - PAddr base = 0; - u64 size = 0; - bool is_free = true; - - PAddr GetEnd() const { - return base + size; - } - - bool CanMergeWith(const FlexibleMemoryArea& next) const { - if (base + size != next.base) { - return false; - } - if (is_free != next.is_free) { - return false; - } - return true; - } -}; - struct VirtualMemoryArea { VAddr base = 0; u64 size = 0; - PAddr phys_base = 0; + std::map phys_areas; VMAType type = VMAType::Free; MemoryProt prot = MemoryProt::NoAccess; - bool disallow_merge = false; std::string name = ""; - uintptr_t fd = 0; - bool is_exec = false; + s32 fd = 0; + bool disallow_merge = false; bool Contains(VAddr addr, u64 size) const { return addr >= base && (addr + size) <= (base + this->size); @@ -141,30 +122,32 @@ struct VirtualMemoryArea { return type != VMAType::Free && type != VMAType::Reserved && type != VMAType::PoolReserved; } - bool CanMergeWith(const VirtualMemoryArea& next) const { + bool CanMergeWith(VirtualMemoryArea& next) { if (disallow_merge || next.disallow_merge) { return false; } if (base + size != next.base) { return false; } - if ((type == VMAType::Direct || type == VMAType::Flexible || type == VMAType::Pooled) && - phys_base + size != next.phys_base) { - return false; + if (type == VMAType::Direct && next.type == VMAType::Direct) { + auto& last_phys = std::prev(phys_areas.end())->second; + auto& first_next_phys = next.phys_areas.begin()->second; + if (last_phys.base + last_phys.size != first_next_phys.base || + last_phys.memory_type != first_next_phys.memory_type) { + return false; + } } if (prot != next.prot || type != next.type) { return false; } + return true; } }; class MemoryManager { - using DMemMap = std::map; - using DMemHandle = DMemMap::iterator; - - using FMemMap = std::map; - using FMemHandle = FMemMap::iterator; + using PhysMap = std::map; + using PhysHandle = PhysMap::iterator; using VMAMap = std::map; using VMAHandle = VMAMap::iterator; @@ -220,10 +203,11 @@ public: // Now make sure the full address range is contained in vma_map. auto vma_handle = FindVMA(virtual_addr); auto addr_to_check = virtual_addr; - s64 size_to_validate = size; + u64 size_to_validate = size; while (vma_handle != vma_map.end() && size_to_validate > 0) { const auto offset_in_vma = addr_to_check - vma_handle->second.base; - const auto size_in_vma = vma_handle->second.size - offset_in_vma; + const auto size_in_vma = + std::min(vma_handle->second.size - offset_in_vma, size_to_validate); size_to_validate -= size_in_vma; addr_to_check += size_in_vma; vma_handle++; @@ -245,7 +229,7 @@ public: void CopySparseMemory(VAddr source, u8* dest, u64 size); - bool TryWriteBacking(void* address, const void* data, u32 num_bytes); + bool TryWriteBacking(void* address, const void* data, u64 size); void SetupMemoryRegions(u64 flexible_size, bool use_extended_mem1, bool use_extended_mem2); @@ -300,34 +284,14 @@ private: return std::prev(vma_map.upper_bound(target)); } - DMemHandle FindDmemArea(PAddr target) { + PhysHandle FindDmemArea(PAddr target) { return std::prev(dmem_map.upper_bound(target)); } - FMemHandle FindFmemArea(PAddr target) { + PhysHandle FindFmemArea(PAddr target) { return std::prev(fmem_map.upper_bound(target)); } - template - Handle MergeAdjacent(auto& handle_map, Handle iter) { - const auto next_vma = std::next(iter); - if (next_vma != handle_map.end() && iter->second.CanMergeWith(next_vma->second)) { - iter->second.size += next_vma->second.size; - handle_map.erase(next_vma); - } - - if (iter != handle_map.begin()) { - auto prev_vma = std::prev(iter); - if (prev_vma->second.CanMergeWith(iter->second)) { - prev_vma->second.size += iter->second.size; - handle_map.erase(iter); - iter = prev_vma; - } - } - - return iter; - } - bool HasPhysicalBacking(VirtualMemoryArea vma) { return vma.type == VMAType::Direct || vma.type == VMAType::Flexible || vma.type == VMAType::Pooled; @@ -335,17 +299,17 @@ private: VAddr SearchFree(VAddr virtual_addr, u64 size, u32 alignment); + VMAHandle MergeAdjacent(VMAMap& map, VMAHandle iter); + + PhysHandle MergeAdjacent(PhysMap& map, PhysHandle iter); + VMAHandle CarveVMA(VAddr virtual_addr, u64 size); - DMemHandle CarveDmemArea(PAddr addr, u64 size); - - FMemHandle CarveFmemArea(PAddr addr, u64 size); + PhysHandle CarvePhysArea(PhysMap& map, PAddr addr, u64 size); VMAHandle Split(VMAHandle vma_handle, u64 offset_in_vma); - DMemHandle Split(DMemHandle dmem_handle, u64 offset_in_area); - - FMemHandle Split(FMemHandle fmem_handle, u64 offset_in_area); + PhysHandle Split(PhysMap& map, PhysHandle dmem_handle, u64 offset_in_area); u64 UnmapBytesFromEntry(VAddr virtual_addr, VirtualMemoryArea vma_base, u64 size); @@ -353,14 +317,15 @@ private: private: AddressSpace impl; - DMemMap dmem_map; - FMemMap fmem_map; + PhysMap dmem_map; + PhysMap fmem_map; VMAMap vma_map; - std::mutex mutex; + Common::SharedFirstMutex mutex{}; u64 total_direct_size{}; u64 total_flexible_size{}; u64 flexible_usage{}; u64 pool_budget{}; + s32 sdk_version{}; Vulkan::Rasterizer* rasterizer{}; struct PrtArea {