From c6b24ec1c74647c89b562b78b1aa06f576204dbb Mon Sep 17 00:00:00 2001 From: TheTurtle <47210458+raphaelthegreat@users.noreply.github.com> Date: Sat, 26 Sep 2026 15:47:11 +0300 Subject: [PATCH] video_core: Renderer optimizations pt1 (#5112) * vk_runtime: Revert to interval sets for barrier tracking Perf capture shows they are faster in almost all cases due to not needing rebalancing after every insert * buffer_cache: Dont use copysparsememory for streaming copies * fetch_shader: Optimize fetch shader parsing * shader_recompiler: Optimize common case of sharp fetch Almost always the sharp will be contigious in flatbuf so just use a single load op to load it * remove zlib * vk_rasterizer: Check dst region not in sync batch before doing cpu fast path * address_space: Maintain sparse page table of backing pointers Can be used to implement CopySparseMemory lock free and cheaply * vk_rasterizer: Proper gds barriers * texture_cache: Bring back lock * fetch_shader: Handle v_mov_b32 * address_space: Avoid out of bounds access with higher than 40bit addresses so stephen doesnt kill me * core: Add setting to control inlining of fetch shader --- CMakeLists.txt | 4 +- LICENSES/Zlib.txt | 11 - .../multi_level_page_table.h | 4 +- src/common/small_vector.h | 12 + src/core/address_space.cpp | 15 + src/core/address_space.h | 14 + src/core/emulator_settings.h | 3 + src/core/memory.cpp | 20 +- .../backend/spirv/spirv_emit_context.cpp | 17 +- .../frontend/fetch_shader.cpp | 128 +++--- src/shader_recompiler/frontend/fetch_shader.h | 22 +- .../frontend/translate/translate.cpp | 50 +-- .../frontend/translate/translate.h | 2 +- .../ir/passes/resource_patching_pass.cpp | 16 +- src/shader_recompiler/resource.h | 18 +- src/shader_recompiler/specialization.h | 19 +- src/video_core/page_manager.cpp | 5 +- .../renderer_vulkan/vk_barrier_tracker.cpp | 368 ------------------ .../renderer_vulkan/vk_barrier_tracker.h | 137 ------- .../renderer_vulkan/vk_graphics_pipeline.cpp | 24 +- .../renderer_vulkan/vk_graphics_pipeline.h | 6 +- .../renderer_vulkan/vk_pipeline_cache.cpp | 24 +- .../renderer_vulkan/vk_pipeline_cache.h | 8 +- .../vk_pipeline_serialization.cpp | 16 +- .../renderer_vulkan/vk_rasterizer.cpp | 23 +- src/video_core/renderer_vulkan/vk_runtime.cpp | 42 +- src/video_core/renderer_vulkan/vk_runtime.h | 14 +- src/video_core/texture_cache/image.cpp | 13 +- src/video_core/texture_cache/texture_cache.h | 6 +- 29 files changed, 310 insertions(+), 731 deletions(-) delete mode 100644 LICENSES/Zlib.txt rename src/{video_core => common}/multi_level_page_table.h (98%) delete mode 100644 src/video_core/renderer_vulkan/vk_barrier_tracker.cpp delete mode 100644 src/video_core/renderer_vulkan/vk_barrier_tracker.h diff --git a/CMakeLists.txt b/CMakeLists.txt index f4febfb42..7087a63dd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -840,6 +840,7 @@ set(COMMON src/common/logging/classes.cpp src/common/number_utils.cpp src/common/memory_patcher.h src/common/memory_patcher.cpp + src/common/multi_level_page_table.h ${CMAKE_CURRENT_BINARY_DIR}/src/common/scm_rev.cpp src/common/scm_rev.h src/common/key_manager.cpp @@ -1118,8 +1119,6 @@ set(VIDEO_CORE src/video_core/amdgpu/cb_db_extent.h src/video_core/buffer_cache/region_manager.h src/video_core/renderer_vulkan/liverpool_to_vk.cpp src/video_core/renderer_vulkan/liverpool_to_vk.h - src/video_core/renderer_vulkan/vk_barrier_tracker.cpp - src/video_core/renderer_vulkan/vk_barrier_tracker.h src/video_core/renderer_vulkan/vk_common.cpp src/video_core/renderer_vulkan/vk_common.h src/video_core/renderer_vulkan/vk_compute_pipeline.cpp @@ -1181,7 +1180,6 @@ set(VIDEO_CORE src/video_core/amdgpu/cb_db_extent.h src/video_core/cache_storage.h src/video_core/page_manager.cpp src/video_core/page_manager.h - src/video_core/multi_level_page_table.h src/video_core/renderdoc.cpp src/video_core/renderdoc.h ) diff --git a/LICENSES/Zlib.txt b/LICENSES/Zlib.txt deleted file mode 100644 index e0e3605ba..000000000 --- a/LICENSES/Zlib.txt +++ /dev/null @@ -1,11 +0,0 @@ -zlib License - -This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source distribution. diff --git a/src/video_core/multi_level_page_table.h b/src/common/multi_level_page_table.h similarity index 98% rename from src/video_core/multi_level_page_table.h rename to src/common/multi_level_page_table.h index de92bc769..b5e80fca3 100644 --- a/src/video_core/multi_level_page_table.h +++ b/src/common/multi_level_page_table.h @@ -7,7 +7,7 @@ #include "common/object_pool.h" #include "common/spin_lock.h" -namespace VideoCore { +namespace Common { template class MultiLevelPageTable final { @@ -87,4 +87,4 @@ private: Common::ObjectPool page_alloc; }; -} // namespace VideoCore +} // namespace Common diff --git a/src/common/small_vector.h b/src/common/small_vector.h index 13a617b6b..876b3b149 100644 --- a/src/common/small_vector.h +++ b/src/common/small_vector.h @@ -85,6 +85,18 @@ public: reset(); } + bool operator==(const SmallVector& other) const { + if (m_size != other.m_size) { + return false; + } + for (size_type i = 0; i < m_size; ++i) { + if (!(m_data[i] == other.m_data[i])) { + return false; + } + } + return true; + } + constexpr reference operator[](size_type i) noexcept { return m_data[i]; } diff --git a/src/core/address_space.cpp b/src/core/address_space.cpp index 2258d7a90..20438e534 100644 --- a/src/core/address_space.cpp +++ b/src/core/address_space.cpp @@ -828,6 +828,14 @@ AddressSpace::AddressSpace() : impl{std::make_unique()} { AddressSpace::~AddressSpace() = default; void* AddressSpace::Map(VAddr virtual_addr, u64 size, PAddr phys_addr, bool is_exec) { + const VAddr end_addr = virtual_addr + size - 1; + if (phys_addr != -1 && std::bit_width(end_addr) <= Traits::ADDRESS_SPACE_BITS) { + const u64 base_page = virtual_addr >> Traits::PAGE_BITS; + for (u64 offset = 0; offset < size; offset += 16_KB) { + backing_pages[base_page + (offset >> Traits::PAGE_BITS)] = + backing_base + phys_addr + offset; + } + } #if ARCH_X86_64 const auto prot = is_exec ? PAGE_EXECUTE_READWRITE : PAGE_READWRITE; #else @@ -849,6 +857,13 @@ void* AddressSpace::MapFile(VAddr virtual_addr, u64 size, u64 offset, u32 prot, } VAddr AddressSpace::Unmap(VAddr virtual_addr, u64* size) { + const VAddr end_addr = virtual_addr + *size - 1; + if (std::bit_width(end_addr) <= Traits::ADDRESS_SPACE_BITS) { + const u64 base_page = virtual_addr >> Traits::PAGE_BITS; + for (u64 offset = 0; offset < *size; offset += 16_KB) { + backing_pages[base_page + (offset >> Traits::PAGE_BITS)] = nullptr; + } + } return impl->Unmap(virtual_addr, size); } diff --git a/src/core/address_space.h b/src/core/address_space.h index 2958fc46f..cd2b24360 100644 --- a/src/core/address_space.h +++ b/src/core/address_space.h @@ -7,6 +7,7 @@ #include #include "common/arch.h" #include "common/enum.h" +#include "common/multi_level_page_table.h" #include "common/types.h" namespace Core { @@ -29,9 +30,21 @@ public: explicit AddressSpace(); ~AddressSpace(); + struct Traits { + using Entry = u8*; + static constexpr size_t ADDRESS_SPACE_BITS = 40; + static constexpr size_t L1_BITS = 16; + static constexpr size_t PAGE_BITS = 14; + static constexpr bool NULL_CHECK = true; + }; + using BackingPageTable = Common::MultiLevelPageTable; + [[nodiscard]] u8* BackingBase() const noexcept { return backing_base; } + [[nodiscard]] const BackingPageTable& BackingPages() const noexcept { + return backing_pages; + } [[nodiscard]] VAddr SystemManagedVirtualBase() noexcept { return reinterpret_cast(system_managed_base); @@ -100,6 +113,7 @@ private: u64 system_reserved_size{}; u8* user_base{}; u64 user_size{}; + BackingPageTable backing_pages; }; } // namespace Core diff --git a/src/core/emulator_settings.h b/src/core/emulator_settings.h index cbecf51da..80bcd4cfb 100644 --- a/src/core/emulator_settings.h +++ b/src/core/emulator_settings.h @@ -438,6 +438,7 @@ struct GPUSettings { Setting rcas_enabled{true}; Setting rcas_attenuation{250}; Setting userfaultfd{false}; + Setting inline_fetch_shader{false}; // TODO add overrides std::vector GetOverrideableFields() const { return std::vector{ @@ -460,6 +461,7 @@ struct GPUSettings { make_override("direct_memory_access_enabled", &GPUSettings::direct_memory_access_enabled), make_override("vblank_frequency", &GPUSettings::vblank_frequency), + make_override("inline_fetch_shader", &GPUSettings::inline_fetch_shader), }; } }; @@ -743,6 +745,7 @@ public: SETTING_FORWARD_BOOL(m_gpu, DirectMemoryAccessEnabled, direct_memory_access_enabled) SETTING_FORWARD_BOOL_READONLY(m_gpu, PatchShaders, patch_shaders) SETTING_FORWARD_BOOL(m_gpu, UserfaultfdTracking, userfaultfd) + SETTING_FORWARD_BOOL_READONLY(m_gpu, InlineFetchShader, inline_fetch_shader) u32 GetVblankFrequency() { if (m_gpu.vblank_frequency.value < 30) { diff --git a/src/core/memory.cpp b/src/core/memory.cpp index e57cee9c8..bf04e8588 100644 --- a/src/core/memory.cpp +++ b/src/core/memory.cpp @@ -145,27 +145,19 @@ void MemoryManager::SetPrtArea(u32 id, VAddr address, u64 size) { } void MemoryManager::CopySparseMemory(VAddr virtual_addr, u8* dest, u64 size) { - std::shared_lock lk{mutex}; - ASSERT_MSG(IsValidMapping(virtual_addr), "Attempted to access invalid address {:#x}", - virtual_addr); - - auto vma = FindVMA(virtual_addr); + const auto& backing_pages = impl.BackingPages(); while (size) { - const u64 offset = virtual_addr - vma->first; - const u64 copy_size = std::min(vma->second.size - (virtual_addr - vma->first), size); - if (vma->second.IsMapped()) { - u8* out = dest; - vma->second.ForEachPhysArea(offset, copy_size, [&](PAddr paddr, u32 chunk_size) { - std::memcpy(out, impl.BackingBase() + paddr, chunk_size); - out += chunk_size; - }); + const u64 page = virtual_addr >> 14; + const u64 offset_in_page = virtual_addr % 16_KB; + const u64 copy_size = std::min(16_KB - offset_in_page, size); + if (auto* entry = backing_pages.find(page); entry && *entry) { + std::memcpy(dest, *entry + offset_in_page, copy_size); } else { std::memset(dest, 0, copy_size); } size -= copy_size; virtual_addr += copy_size; dest += copy_size; - ++vma; } } diff --git a/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp b/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp index 5f3ae90e9..09c53cf98 100644 --- a/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp +++ b/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp @@ -332,21 +332,22 @@ void EmitContext::DefineInputs() { DefineVariable(U32[1], spv::BuiltIn::BaseInstance, spv::StorageClass::Input); } - const auto fetch_shader = Gcn::ParseFetchShader(info); - if (!fetch_shader) { + Shader::Gcn::FetchShaderData fetch_shader; + if (!Gcn::ParseFetchShader(info, fetch_shader)) { break; } - for (const auto& attrib : fetch_shader->attributes) { - ASSERT(attrib.semantic < IR::NumParams); + ASSERT(fetch_shader.attributes.size() <= IR::NumParams); + for (u32 semantic = 0; semantic < fetch_shader.attributes.size(); ++semantic) { + const auto& attrib = fetch_shader.attributes[semantic]; const auto sharp = attrib.GetSharp(info); const Id type{GetAttributeType(*this, sharp.GetNumberFmt())[4]}; - Id id{DefineInput(type, attrib.semantic)}; + Id id{DefineInput(type, semantic)}; if (attrib.GetStepRate() != Gcn::VertexAttribute::InstanceIdType::None) { - Name(id, fmt::format("vs_instance_attr{}", attrib.semantic)); + Name(id, fmt::format("vs_instance_attr{}", semantic)); } else { - Name(id, fmt::format("vs_in_attr{}", attrib.semantic)); + Name(id, fmt::format("vs_in_attr{}", semantic)); } - input_params[attrib.semantic] = GetAttributeInfo(sharp.GetNumberFmt(), id, 4, false); + input_params[semantic] = GetAttributeInfo(sharp.GetNumberFmt(), id, 4, false); } break; } diff --git a/src/shader_recompiler/frontend/fetch_shader.cpp b/src/shader_recompiler/frontend/fetch_shader.cpp index ba0635546..c48e91cdc 100644 --- a/src/shader_recompiler/frontend/fetch_shader.cpp +++ b/src/shader_recompiler/frontend/fetch_shader.cpp @@ -1,9 +1,11 @@ // SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include #include "common/assert.h" #include "shader_recompiler/frontend/decode.h" #include "shader_recompiler/frontend/fetch_shader.h" +#include "shader_recompiler/frontend/opcodes.h" namespace Shader::Gcn { @@ -32,28 +34,18 @@ namespace Shader::Gcn { * We take the reverse way, extract the original input semantics from these instructions. **/ -static bool IsTypedBufferLoad(const Gcn::GcnInst& inst) { - return inst.opcode == Opcode::TBUFFER_LOAD_FORMAT_X || - inst.opcode == Opcode::TBUFFER_LOAD_FORMAT_XY || - inst.opcode == Opcode::TBUFFER_LOAD_FORMAT_XYZ || - inst.opcode == Opcode::TBUFFER_LOAD_FORMAT_XYZW; -} - const u32* GetFetchShaderCode(const Info& info, u32 sgpr_base) { const u32* code; std::memcpy(&code, &info.user_data[sgpr_base], sizeof(code)); return code; } -std::optional ParseFetchShader(const Shader::Info& info) { +bool ParseFetchShader(const Shader::Info& info, FetchShaderData& out_fetch_data) { if (!info.has_fetch_shader) { - return std::nullopt; + return false; } - const auto* code = GetFetchShaderCode(info, info.fetch_shader_sgpr_base); - FetchShaderData data{}; - GcnCodeSlice code_slice(code, code + std::numeric_limits::max()); - GcnDecodeContext decoder; + out_fetch_data.attributes.clear(); struct VsharpLoad { u32 dword_offset{}; @@ -61,54 +53,86 @@ std::optional ParseFetchShader(const Shader::Info& info) { }; std::array loads{}; - u32 semantic_index = 0; - while (!code_slice.atEnd()) { - const auto inst = decoder.decodeInstruction(code_slice); - data.size += inst.length; + const auto* code = GetFetchShaderCode(info, info.fetch_shader_sgpr_base); + const u32* ptr = code; + while (true) { + const u32 word0 = *(ptr++); + const auto encoding = GetInstructionEncoding(word0); - if (inst.opcode == Opcode::S_SETPC_B64) { + if (encoding == InstEncoding::SOP1) { + const auto opcode = OpcodeSOP1((word0 >> 8) & 0xff); + ASSERT(opcode == OpcodeSOP1::S_SETPC_B64); break; - } - - if (inst.inst_class == InstClass::ScalarMemRd) { - loads[inst.dst[0].code] = VsharpLoad{inst.control.smrd.offset, inst.src[0].code * 2}; - continue; - } - - if (inst.opcode == Opcode::V_ADD_I32) { - const auto vgpr = inst.dst[0].code; - const auto sgpr = s8(inst.src[0].code); - switch (vgpr) { - case 0: // V0 is always the vertex offset - data.vertex_offset_sgpr = sgpr; - break; - case 3: // V3 is always the instance offset - data.instance_offset_sgpr = sgpr; - break; - default: + } else if (encoding == InstEncoding::SMRD) { + const auto opcode = OpcodeSMRD((word0 >> 22) & 0x1f); + const u32 sdst = (word0 >> 15) & 0x7f; + const u32 sbase = (word0 >> 9) & 0x3f; + const u32 offset = word0 & 0xff; + ASSERT(opcode == OpcodeSMRD::S_LOAD_DWORDX4); + loads[sdst] = VsharpLoad{offset, sbase * 2}; + } else if (encoding == InstEncoding::VOP2) { + const auto opcode = OpcodeVOP2((word0 >> 25) & 0x3f); + const u32 src0 = word0 & 0x1ff; + const u32 vsrc1 = (word0 >> 9) & 0xff; + const u32 vdst = (word0 >> 17) & 0xff; + ASSERT(opcode == OpcodeVOP2::V_ADD_I32 && src0 < 16 && vsrc1 == vdst); + if (vdst == 0) { + // V0 is always the vertex offset + out_fetch_data.vertex_offset_sgpr = src0; + } else if (vdst == 3) { + // V3 is always the instance offset + out_fetch_data.instance_offset_sgpr = src0; + } else { UNREACHABLE(); } - } - - if (inst.inst_class == InstClass::VectorMemBufFmt) { + } else if (encoding == InstEncoding::MUBUF || encoding == InstEncoding::MTBUF) { // SRSRC is in units of 4 SPGRs while SBASE is in pairs of SGPRs - const u32 base_sgpr = inst.src[2].code * 4; - auto& attrib = data.attributes.emplace_back(); - attrib.semantic = semantic_index++; - attrib.dest_vgpr = inst.src[1].code; - attrib.num_elements = inst.control.mubuf.count; - attrib.sgpr_base = loads[base_sgpr].base_sgpr; - attrib.dword_offset = loads[base_sgpr].dword_offset; - attrib.inst_offset = inst.control.mtbuf.offset; - attrib.instance_data = inst.src[0].code; - if (IsTypedBufferLoad(inst)) { - attrib.data_format = inst.control.mtbuf.dfmt; - attrib.num_format = inst.control.mtbuf.nfmt; + const u32 word1 = *(ptr++); + const u32 srsrc = ((word1 >> 16) & 0x1f) * 4; + auto& attrib = out_fetch_data.attributes.emplace_back(); + attrib.dest_vgpr = (word1 >> 8) & 0xff; + if (encoding == InstEncoding::MUBUF) { + const auto opcode = OpcodeMUBUF((word0 >> 18) & 0x7f); + ASSERT(opcode >= OpcodeMUBUF::BUFFER_LOAD_FORMAT_X && + opcode <= OpcodeMUBUF::BUFFER_LOAD_FORMAT_XYZW); + attrib.num_elements = (std::to_underlying(opcode) - + std::to_underlying(OpcodeMUBUF::BUFFER_LOAD_FORMAT_X)) + + 1; + } else { + const auto opcode = OpcodeMTBUF((word0 >> 16) & 0x7); + ASSERT(opcode >= OpcodeMTBUF::TBUFFER_LOAD_FORMAT_X && + opcode <= OpcodeMTBUF::TBUFFER_LOAD_FORMAT_XYZW); + attrib.num_elements = (std::to_underlying(opcode) - + std::to_underlying(OpcodeMTBUF::TBUFFER_LOAD_FORMAT_X)) + + 1; } + attrib.sgpr_base = loads[srsrc].base_sgpr; + attrib.dword_offset = loads[srsrc].dword_offset; + attrib.inst_offset = word0 & 0xfff; + attrib.instance_data = word1 & 0xff; + if (encoding == InstEncoding::MTBUF) { + attrib.data_format = (word0 >> 19) & 0xf; + attrib.num_format = (word0 >> 23) & 0x7; + } else { + attrib.data_format = 0; + attrib.num_format = 0; + } + } else if (encoding == InstEncoding::SOPP) { + const auto opcode = OpcodeSOPP((word0 >> 16) & 0x7f); + ASSERT(opcode == OpcodeSOPP::S_WAITCNT); + } else if (encoding == InstEncoding::VOP1) { + const auto opcode = OpcodeVOP1((word0 >> 9) & 0xff); + const u32 src0 = word0 & 0x1ff; + const u32 vdst = (word0 >> 17) & 0xff; + ASSERT(opcode == OpcodeVOP1::V_MOV_B32 && src0 == 242); + LOG_WARNING(Render_Recompiler, "Fetch shader has V{} = 1.0 which is ignored", vdst); + } else { + UNREACHABLE_MSG("Unexpected instruction encoding in fetch shader {}", u32(encoding)); } } - return data; + out_fetch_data.size = (ptr - code) * sizeof(u32); + return true; } } // namespace Shader::Gcn diff --git a/src/shader_recompiler/frontend/fetch_shader.h b/src/shader_recompiler/frontend/fetch_shader.h index a57bbb0c6..66c78c44a 100644 --- a/src/shader_recompiler/frontend/fetch_shader.h +++ b/src/shader_recompiler/frontend/fetch_shader.h @@ -3,8 +3,7 @@ #pragma once -#include -#include +#include "common/small_vector.h" #include "common/types.h" #include "shader_recompiler/info.h" @@ -22,15 +21,14 @@ struct VertexAttribute { Plain = 3, }; - u8 semantic; ///< Semantic index of the attribute u8 dest_vgpr; ///< Destination VGPR to load first component. u8 num_elements; ///< Number of components to load u8 sgpr_base; ///< SGPR that contains the pointer to the list of vertex V# u8 dword_offset; ///< The dword offset of the V# that describes this attribute. u8 instance_data; ///< Indicates that the buffer will be accessed in instance rate u8 inst_offset; ///< Instruction offset applied on the formatted buffer loads - u8 data_format{}; ///< Data format override when typed buffer loads are used - u8 num_format{}; ///< Number format override when typed buffer loads are used + u8 data_format; ///< Data format override when typed buffer loads are used + u8 num_format; ///< Number format override when typed buffer loads are used InstanceIdType GetStepRate() const { return static_cast(instance_data); @@ -46,16 +44,12 @@ struct VertexAttribute { return buffer; } - bool operator==(const VertexAttribute& other) const { - return semantic == other.semantic && dest_vgpr == other.dest_vgpr && - num_elements == other.num_elements && sgpr_base == other.sgpr_base && - dword_offset == other.dword_offset && instance_data == other.instance_data; - } + bool operator==(const VertexAttribute& other) const = default; }; struct FetchShaderData { u32 size = 0; - std::vector attributes; + SmallVector attributes; s8 vertex_offset_sgpr = -1; ///< SGPR of vertex offset from VADDR s8 instance_offset_sgpr = -1; ///< SGPR of instance offset from VADDR @@ -64,12 +58,16 @@ struct FetchShaderData { instance_offset_sgpr == other.instance_offset_sgpr; } + bool Empty() const { + return size == 0; + } + void Serialize(Serialization::Archive& ar) const; bool Deserialize(Serialization::Archive& buffer); }; const u32* GetFetchShaderCode(const Info& info, u32 sgpr_base); -std::optional ParseFetchShader(const Shader::Info& info); +bool ParseFetchShader(const Shader::Info& info, FetchShaderData& out_fetch_data); } // namespace Shader::Gcn diff --git a/src/shader_recompiler/frontend/translate/translate.cpp b/src/shader_recompiler/frontend/translate/translate.cpp index 49befa76e..f34e0aa5a 100644 --- a/src/shader_recompiler/frontend/translate/translate.cpp +++ b/src/shader_recompiler/frontend/translate/translate.cpp @@ -4,6 +4,7 @@ #include "common/io_file.h" #include "common/path_util.h" #include "core/emulator_settings.h" +#include "shader_recompiler/frontend/decode.h" #include "shader_recompiler/frontend/fetch_shader.h" #include "shader_recompiler/frontend/translate/translate.h" #include "shader_recompiler/info.h" @@ -112,10 +113,10 @@ void Translator::EmitPrologue(IR::Block* first_block) { // v0: vertex ID, always present IR::U32 vertex_id = ir.GetAttributeU32(IR::Attribute::VertexId); if (base_vertex_sgpr != -1) { - if (!fetch_data || fetch_data->vertex_offset_sgpr == -1) { + if (!fetch_data.Empty() || fetch_data.vertex_offset_sgpr == -1) { vertex_id = ir.ISub(vertex_id, ir.GetAttributeU32(IR::Attribute::BaseVertex)); } else { - ASSERT_MSG(fetch_data->vertex_offset_sgpr == base_vertex_sgpr, + ASSERT_MSG(fetch_data.vertex_offset_sgpr == base_vertex_sgpr, "Fetch shader in indirect draw uses wrong base vertex"); } } @@ -157,11 +158,11 @@ void Translator::EmitPrologue(IR::Block* first_block) { if (runtime_info.props.num_input_vgprs > 2) { IR::U32 instance_id = ir.GetAttributeU32(IR::Attribute::InstanceId); if (base_instance_sgpr != -1) { - if (!fetch_data || fetch_data->instance_offset_sgpr == -1) { + if (!fetch_data.Empty() || fetch_data.instance_offset_sgpr == -1) { instance_id = ir.ISub(instance_id, ir.GetAttributeU32(IR::Attribute::BaseInstance)); } else { - ASSERT_MSG(fetch_data->instance_offset_sgpr == base_instance_sgpr, + ASSERT_MSG(fetch_data.instance_offset_sgpr == base_instance_sgpr, "Fetch shader in indirect draw uses wrong base instance"); } } @@ -1134,29 +1135,27 @@ template void Translator::SetDstPk(const InstOperand& operand, void Translator::EmitFetch(const GcnInst& inst) { const auto code_sgpr_base = inst.src[0].code; -#if 0 - // Translate fetch shader inline using regular buffer bindings; useful for debugging. - const auto* code = GetFetchShaderCode(info, code_sgpr_base); - GcnCodeSlice slice(code, code + std::numeric_limits::max()); - GcnDecodeContext decoder; + if (EmulatorSettings.IsInlineFetchShader()) { + // Translate fetch shader inline using regular buffer bindings; useful for debugging. + const auto* code = GetFetchShaderCode(info, code_sgpr_base); + GcnCodeSlice slice(code, code + std::numeric_limits::max()); + GcnDecodeContext decoder; - // Decode and save instructions - while (!slice.atEnd()) { - const auto sub_inst = decoder.decodeInstruction(slice); - if (sub_inst.opcode == Opcode::S_SETPC_B64) { - // Assume we're swapping back to the main shader. - break; + // Decode and save instructions + while (!slice.atEnd()) { + const auto sub_inst = decoder.decodeInstruction(slice); + if (sub_inst.opcode == Opcode::S_SETPC_B64) { + // Assume we're swapping back to the main shader. + break; + } + TranslateInstruction(sub_inst); } - TranslateInstruction(sub_inst); + return; } - return; -#endif info.has_fetch_shader = true; info.fetch_shader_sgpr_base = code_sgpr_base; - - fetch_data = ParseFetchShader(info); - ASSERT(fetch_data.has_value()); + ASSERT(ParseFetchShader(info, fetch_data)); if (EmulatorSettings.IsDumpShaders()) { using namespace Common::FS; @@ -1167,11 +1166,12 @@ void Translator::EmitFetch(const GcnInst& inst) { const auto filename = fmt::format("vs_{:#018x}.fetch.bin", info.pgm_hash); const auto file = IOFile{dump_dir / filename, FileAccessMode::Create}; const auto* code = GetFetchShaderCode(info, code_sgpr_base); - file.WriteRaw(code, fetch_data->size); + file.WriteRaw(code, fetch_data.size); } - for (const auto& attrib : fetch_data->attributes) { - const IR::Attribute attr{IR::Attribute::Param0 + attrib.semantic}; + for (u32 semantic = 0; semantic < fetch_data.attributes.size(); ++semantic) { + const auto& attrib = fetch_data.attributes[semantic]; + const IR::Attribute attr{IR::Attribute::Param0 + semantic}; IR::VectorReg dst_reg{attrib.dest_vgpr}; // Read the V# of the attribute to figure out component number and type. @@ -1182,7 +1182,7 @@ void Translator::EmitFetch(const GcnInst& inst) { const auto converted = IR::ApplyReadNumberConversionVec4(ir, values, buffer.GetNumberConversion()); const auto swizzled = ApplySwizzle(ir, converted, buffer.DstSelect()); - for (u32 i = 0; i < 4; i++) { + for (u32 i = 0; i < attrib.num_elements; i++) { ir.SetVectorReg(dst_reg++, IR::F32{ir.CompositeExtract(swizzled, i)}); } } diff --git a/src/shader_recompiler/frontend/translate/translate.h b/src/shader_recompiler/frontend/translate/translate.h index 92c137998..22aad7952 100644 --- a/src/shader_recompiler/frontend/translate/translate.h +++ b/src/shader_recompiler/frontend/translate/translate.h @@ -393,7 +393,7 @@ private: u32 next_vgpr_num; std::unordered_map vgpr_map; std::array vgpr_to_interp{}; - std::optional fetch_data{}; + FetchShaderData fetch_data{}; bool opcode_missing = false; u32 pc{}; }; diff --git a/src/shader_recompiler/ir/passes/resource_patching_pass.cpp b/src/shader_recompiler/ir/passes/resource_patching_pass.cpp index 860281621..ce48e8572 100644 --- a/src/shader_recompiler/ir/passes/resource_patching_pass.cpp +++ b/src/shader_recompiler/ir/passes/resource_patching_pass.cpp @@ -4,15 +4,14 @@ #include #include "shader_recompiler/info.h" #include "shader_recompiler/ir/basic_block.h" -#include "shader_recompiler/ir/breadth_first_search.h" #include "shader_recompiler/ir/ir_emitter.h" #include "shader_recompiler/ir/opcodes.h" #include "shader_recompiler/ir/operand_helper.h" #include "shader_recompiler/ir/passes/ir_passes.h" #include "shader_recompiler/ir/passes/resource_pass.h" -#include "shader_recompiler/ir/program.h" #include "shader_recompiler/ir/reinterpret.h" #include "shader_recompiler/profile.h" +#include "shader_recompiler/resource.h" #include "video_core/amdgpu/resource.h" namespace Shader::Optimization { @@ -181,6 +180,7 @@ SharpLocation SharpLocationFromSource(const IR::Inst* inst) { template SharpFetch ConstructSharpFetch(const SharpReference& sharp) { + using Summary = SharpFetch::Summary; SharpFetch sharp_fetch{}; for (u32 i = 0; i < sharp.num_dwords; i++) { auto dword = sharp.dwords[i]; @@ -189,8 +189,20 @@ SharpFetch ConstructSharpFetch(const SharpReference& sharp) { } else { sharp_fetch.offsets[i] = SharpLocationFromSource(dword.Inst()); sharp_fetch.load_mask |= (1 << i); + if (sharp_fetch.offsets[i] == UNKNOWN_LOCATION) { + sharp_fetch.summary = Summary::Invalid; + } } } + if (sharp_fetch.summary != Summary::Invalid) { + const u32 base = sharp_fetch.offsets[0]; + for (u32 i = 1; i < sharp.num_dwords; ++i) { + if (sharp_fetch.offsets[i] - base != i) { + return sharp_fetch; + } + } + } + sharp_fetch.summary = Summary::SingleLoad; return sharp_fetch; } diff --git a/src/shader_recompiler/resource.h b/src/shader_recompiler/resource.h index ff5bff8a9..2c50b0ddb 100644 --- a/src/shader_recompiler/resource.h +++ b/src/shader_recompiler/resource.h @@ -26,6 +26,13 @@ struct SharpFetch { static constexpr std::size_t N = sizeof(T) / sizeof(u32); static_assert(N <= 8); + enum class Summary : u8 { + SingleLoad, + MultiLoad, + Invalid, + }; + + Summary summary{Summary::MultiLoad}; std::array immediates; std::array offsets; u8 load_mask; @@ -35,12 +42,13 @@ struct SharpFetch { template requires(num_dwords <= N) constexpr bool Fetch(const u32* flatbuf, T* out) const { - u8 mask = load_mask; - for (u32 i = 0; i < num_dwords; i++) { - if (offsets[i] == UNKNOWN_LOCATION) { - return false; - } + if (summary == Summary::SingleLoad) [[likely]] { + std::memcpy(out, flatbuf + offsets[0], num_dwords * sizeof(u32)); + return true; + } else if (summary == Summary::Invalid) [[unlikely]] { + return false; } + u8 mask = load_mask; std::array out_dw; for (u32 i = 0; i < num_dwords; i++) { out_dw[i] = (mask & 1) ? flatbuf[offsets[i]] : immediates[i]; diff --git a/src/shader_recompiler/specialization.h b/src/shader_recompiler/specialization.h index 75b9dc961..76f21d03b 100644 --- a/src/shader_recompiler/specialization.h +++ b/src/shader_recompiler/specialization.h @@ -83,22 +83,21 @@ struct StageSpecialization { const Info* info{}; RuntimeInfo runtime_info{}; std::bitset bitset{}; - std::optional fetch_shader_data{}; - boost::container::small_vector vs_attribs; - boost::container::small_vector buffers; - boost::container::small_vector images; - boost::container::small_vector fmasks; - boost::container::small_vector samplers; + Gcn::FetchShaderData fetch_shader_data{}; + SmallVector vs_attribs; + SmallVector buffers; + SmallVector images; + SmallVector fmasks; + SmallVector samplers; Backend::Bindings start{}; StageSpecialization() = default; StageSpecialization(const Info& info_, RuntimeInfo runtime_info_, const Profile& profile_, Backend::Bindings start_) : info{&info_}, runtime_info{runtime_info_}, start{start_} { - fetch_shader_data = Gcn::ParseFetchShader(info_); - if (info_.sw_stage == SwStage::Vertex && fetch_shader_data) { + if (info_.sw_stage == SwStage::Vertex && Gcn::ParseFetchShader(info_, fetch_shader_data)) { // Specialize shader on VS input number types to follow spec. - ForEachSharp(vs_attribs, fetch_shader_data->attributes, + ForEachSharp(vs_attribs, fetch_shader_data.attributes, [this](auto& spec, const auto& desc, AmdGpu::Buffer sharp) { using InstanceIdType = Shader::Gcn::VertexAttribute::InstanceIdType; if (const auto step_rate = desc.GetStepRate(); @@ -183,7 +182,7 @@ struct StageSpecialization { binding++; continue; } - bitset.set(binding++); + bitset[binding++] = true; func(spec, desc, sharp); } } diff --git a/src/video_core/page_manager.cpp b/src/video_core/page_manager.cpp index b3a386015..49f86394b 100644 --- a/src/video_core/page_manager.cpp +++ b/src/video_core/page_manager.cpp @@ -7,6 +7,7 @@ #include "common/debug.h" #include "common/div_ceil.h" #include "common/error.h" +#include "common/multi_level_page_table.h" #include "common/signal_context.h" #include "common/thread.h" #include "core/emulator_settings.h" @@ -260,7 +261,7 @@ struct PageManager::Impl { static constexpr size_t PAGE_BITS = PM_PAGE_BITS; static constexpr bool NULL_CHECK = false; }; - MultiLevelPageTable cached_pages; + Common::MultiLevelPageTable cached_pages; struct MutexTraits { #ifdef PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP using Entry = Common::AdaptiveMutex; @@ -272,7 +273,7 @@ struct PageManager::Impl { static constexpr size_t PAGE_BITS = PM_PAGE_BITS; static constexpr bool NULL_CHECK = false; }; - MultiLevelPageTable locks; + Common::MultiLevelPageTable locks; }; #ifdef __linux__ diff --git a/src/video_core/renderer_vulkan/vk_barrier_tracker.cpp b/src/video_core/renderer_vulkan/vk_barrier_tracker.cpp deleted file mode 100644 index c7d9131d2..000000000 --- a/src/video_core/renderer_vulkan/vk_barrier_tracker.cpp +++ /dev/null @@ -1,368 +0,0 @@ -// SPDX-FileCopyrightText: Copyright DXVK Project -// SPDX-License-Identifier: Zlib - -#include -#include "video_core/renderer_vulkan/vk_barrier_tracker.h" - -namespace Vulkan { - -BarrierTracker::BarrierTracker() { - // Having an accessible 0 node makes certain things easier to - // implement and allows us to use 0 as an invalid node index. - nodes.emplace_back(); - - // Pre-allocate root nodes for the implicit hash table - for (u32 i = 0; i < 2u * HashTableSize; i++) { - AllocateNode(); - } -} - -bool BarrierTracker::FindRange(const AddressRange& range, Access access_type) const { - u32 root_index = ComputeRootIndex(range, access_type); - u32 node_index = FindNode(range, root_index); - return node_index; -} - -void BarrierTracker::InsertRange(const AddressRange& range, Access access_type) { - // If we can just insert the node with no conflicts, we don't have to do anything. - u32 root_index = ComputeRootIndex(range, access_type); - u32 node_index = InsertNode(range, root_index); - - if (!node_index) [[likely]] { - return; - } - - // If there's an existing node and it contains the entire range we want to add already, also - // don't do anything. If there are conflicting access ops, reset it. - auto& node = nodes[node_index]; - if (node.address_range.Contains(range)) { - return; - } - - // Otherwise, check if there are any other overlapping ranges. - // If that is not the case, simply update the range we found. - bool has_overlap = false; - - if (range.range_start < node.address_range.range_start) { - AddressRange test_range; - test_range.resource = range.resource; - test_range.range_start = range.range_start; - test_range.range_end = node.address_range.range_start - 1; - has_overlap = FindNode(test_range, root_index); - } - - if (range.range_end > node.address_range.range_end && !has_overlap) { - AddressRange test_range; - test_range.resource = range.resource; - test_range.range_start = node.address_range.range_end + 1; - test_range.range_end = range.range_end; - has_overlap = FindNode(test_range, root_index); - } - - if (!has_overlap) { - node.address_range.range_start = - std::min(node.address_range.range_start, range.range_start); - node.address_range.range_end = std::max(node.address_range.range_end, range.range_end); - return; - } - - // If there are multiple ranges overlapping the one being inserted, remove them all and insert - // the merged range. - AddressRange merged_range = range; - while (node_index) { - auto& node = nodes[node_index]; - merged_range.range_start = - std::min(merged_range.range_start, node.address_range.range_start); - merged_range.range_end = std::max(merged_range.range_end, node.address_range.range_end); - - RemoveNode(node_index, root_index); - node_index = FindNode(range, root_index); - } - InsertNode(merged_range, root_index); -} - -void BarrierTracker::Clear() { - root_mask_valid = 0; - while (root_mask_subtree) { - // Free subtrees if any, but keep the root node intact - const u32 root_index = std::countr_zero(root_mask_subtree) + 1; - auto& root = nodes[root_index]; - if (root.header) { - FreeNode(root.Child(0)); - FreeNode(root.Child(1)); - root.header = 0; - } - root_mask_subtree &= root_mask_subtree - 1; - } -} - -u32 BarrierTracker::AllocateNode() { - if (!free.empty()) { - const u32 node_index = free.back(); - free.pop_back(); - - // Free any subtree that the node might still have - auto& node = nodes[node_index]; - FreeNode(node.Child(0)); - FreeNode(node.Child(1)); - - node.header = 0; - return node_index; - } else { - // Allocate entirely new node in the array - const u32 node_index = nodes.size(); - nodes.emplace_back(); - return node_index; - } -} - -void BarrierTracker::FreeNode(u32 node) { - if (node) { - free.push_back(node); - } -} - -u32 BarrierTracker::FindNode(const AddressRange& range, u32 root_index) const { - // Check if the given root is valid at all - const u64 root_bit = 1ULL << (root_index - 1); - if (!(root_mask_valid & root_bit)) { - return false; - } - - // Traverse search tree normally - u32 node_index = root_index; - while (node_index) { - auto& node = nodes[node_index]; - if (node.address_range.Overlaps(range)) { - return node_index; - } - node_index = node.Child(node.address_range < range); - } - return 0; -} - -u32 BarrierTracker::InsertNode(const AddressRange& range, u32 root_index) { - // Check if the given root is valid at all - const u64 root_bit = 1ULL << (root_index - 1); - - if (!(root_mask_valid & root_bit)) { - // Update root node as necessary. Also reset its red-ness if we set it during deletion. - root_mask_valid |= root_bit; - auto& node = nodes[root_index]; - node.header = 0; - node.address_range = range; - return 0; - } - - // Traverse tree and abort if we find any range overlapping the one we're trying to insert. - u32 parent_index = root_index; - u32 child_index = 0; - - while (true) { - auto& parent = nodes[parent_index]; - if (parent.address_range.Overlaps(range)) { - return parent_index; - } - child_index = parent.address_range < range; - if (!parent.Child(child_index)) { - break; - } - parent_index = parent.Child(child_index); - } - - // Create and insert new node into the tree - const u32 node_index = AllocateNode(); - - auto& parent = nodes[parent_index]; - parent.SetChild(child_index, node_index); - - auto& node = nodes[node_index]; - node.SetRed(true); - node.SetParent(parent_index); - node.address_range = range; - - // Only do the fixup to maintain red-black properties if - // we haven't marked the root node as red in a deletion. - if (parent_index != root_index && !nodes[root_index].IsRed()) { - RebalancePostInsert(node_index, root_index); - } - - root_mask_subtree |= root_bit; - return 0; -} - -void BarrierTracker::RemoveNode(u32 node_index, u32 root_index) { - auto& node = nodes[node_index]; - - u32 l = node.Child(0); - u32 r = node.Child(1); - - if (l && r) { - // Both children are valid. Take the payload from the smallest - // node in the right subtree and delete that node instead. - u32 child_index = r; - while (nodes[child_index].Child(0)) { - child_index = nodes[child_index].Child(0); - } - - node.address_range = nodes[child_index].address_range; - RemoveNode(child_index, root_index); - return; - } - - // Deletion is expected to be exceptionally rare, to the point of - // being irrelevant in practice since it can only ever happen if an - // app reads multiple disjoint blocks of a resource and then reads - // another range covering multiple of those blocks again. Instead - // of implementing a complex post-delete fixup, mark the root as - // red and allow the tree to go unbalanced until the next reset. - if (!node.IsRed() && (node_index != root_index)) { - nodes[root_index].SetRed(true); - } - - // We're deleting the a node with one or no children. To avoid - // special-casing the root node, copy the child node to it and - // update links as necessary. - u32 child_index = std::max(l, r); - u32 parent_index = node.Parent(); - - if (child_index) { - auto& child = nodes[child_index]; - - u32 cl = child.Child(0); - u32 cr = child.Child(1); - - node.SetChild(0, cl); - node.SetChild(1, cr); - - if (node_index != root_index) - node.SetRed(child.IsRed()); - - node.address_range = child.address_range; - - if (cl) { - nodes[cl].SetParent(node_index); - } - if (cr) { - nodes[cr].SetParent(node_index); - } - - child.header = 0; - FreeNode(child_index); - } else if (node_index != root_index) { - // Removing leaf node, update parent link and move on. - auto& parent = nodes[parent_index]; - - const u32 which = parent.Child(1) == node_index; - parent.SetChild(which, 0); - - node.header = 0; - FreeNode(node_index); - } else { - // Removing root with no children, mark tree as invalid - const u64 root_bit = 1ULL << (root_index - 1u); - root_mask_subtree &= ~root_bit; - root_mask_valid &= ~root_bit; - } -} - -void BarrierTracker::RebalancePostInsert(u32 node_index, u32 root_index) { - while (node_index != root_index) { - auto& node = nodes[node_index]; - auto& p = nodes[node.Parent()]; - - if (!p.IsRed()) { - break; - } - - auto& g = nodes[p.Parent()]; - - if (g.Child(1) == node.Parent()) { - auto& u = nodes[g.Child(0)]; - - if (g.Child(0) && u.IsRed()) { - g.SetRed(true); - u.SetRed(false); - p.SetRed(false); - node_index = p.Parent(); - } else { - if (p.Child(0) == node_index) { - RotateRight(node.Parent(), root_index); - } - p.SetRed(false); - g.SetRed(true); - RotateLeft(p.Parent(), root_index); - } - } else { - auto& u = nodes[g.Child(1)]; - if (g.Child(1) && u.IsRed()) { - g.SetRed(true); - u.SetRed(false); - p.SetRed(false); - node_index = p.Parent(); - } else { - if (p.Child(1) == node_index) { - RotateLeft(node.Parent(), root_index); - } - p.SetRed(false); - g.SetRed(true); - RotateRight(p.Parent(), root_index); - } - } - } - nodes[root_index].SetRed(false); -} - -void BarrierTracker::RotateLeft(u32 node_index, u32 root_index) { - // This implements rotations in such a way that the node to - // rotate around does not move. This is important to avoid - // having a special case for the root node, and avoids having - // to access the parent or special-case the root node. - auto& node = nodes[node_index]; - - u32 l = node.Child(0); - u32 r = node.Child(1); - u32 rl = nodes[r].Child(0); - u32 rr = nodes[r].Child(1); - - nodes[l].SetParent(r); - - const bool is_red = nodes[r].IsRed(); - nodes[r].SetRed(node.IsRed()); - nodes[r].SetChild(0, l); - nodes[r].SetChild(1, rl); - - nodes[rr].SetParent(node_index); - - node.SetRed(is_red && node_index != root_index); - node.SetChild(0, r); - node.SetChild(1, rr); - - std::swap(node.address_range, nodes[r].address_range); -} - -void BarrierTracker::RotateRight(u32 node_index, u32 root_index) { - auto& node = nodes[node_index]; - - u32 l = node.Child(0); - u32 r = node.Child(1); - u32 ll = nodes[l].Child(0); - u32 lr = nodes[l].Child(1); - - nodes[r].SetParent(l); - - const bool is_red = nodes[l].IsRed(); - nodes[l].SetRed(node.IsRed()); - nodes[l].SetChild(0, lr); - nodes[l].SetChild(1, r); - - nodes[ll].SetParent(node_index); - - node.SetRed(is_red && node_index != root_index); - node.SetChild(0, ll); - node.SetChild(1, l); - - std::swap(node.address_range, nodes[l].address_range); -} - -} // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_barrier_tracker.h b/src/video_core/renderer_vulkan/vk_barrier_tracker.h deleted file mode 100644 index 0a41c812a..000000000 --- a/src/video_core/renderer_vulkan/vk_barrier_tracker.h +++ /dev/null @@ -1,137 +0,0 @@ -// SPDX-FileCopyrightText: Copyright DXVK Project -// SPDX-License-Identifier: Zlib - -#pragma once - -#include -#include "common/types.h" - -namespace Vulkan { - -enum class Access : u32 { - None = 0, - Read = 1, - Write = 2, -}; - -struct AddressRange { - u64 resource{}; - u64 range_start{}; - u64 range_end{}; - - bool Contains(const AddressRange& other) const { - return resource == other.resource && range_start <= other.range_start && - range_end >= other.range_end; - } - - bool Overlaps(const AddressRange& other) const { - return resource == other.resource && range_end >= other.range_start && - range_start <= other.range_end; - } - - bool operator<(const AddressRange& other) const { - return (resource < other.resource) || - (resource == other.resource && range_start < other.range_start); - } -}; - -/** - * Node of a red-black tree, consisting of a packed node header as well as a resource address range. - * GCC generates weird code with bitfields here, so pack manually. - */ -struct BarrierTreeNode { - constexpr static u64 NodeIndexMask = (1u << 21) - 1u; - - // Packed header with node indices and the node color. - // [0:0]: Set if the node is red, clear otherwise. - // [21:1]: Index of the left child node, may be 0. - // [42:22]: Index of the right child node, may be 0. - // [43:63]: Index of the parent node, may be 0 for the root. - u64 header{}; - - // Address range of the node - AddressRange address_range{}; - - void SetRed(bool red) { - header &= ~1ULL; - header |= u64(red); - } - - bool IsRed() const { - return header & 1u; - } - - void SetParent(u32 node) { - header &= ~(NodeIndexMask << 43); - header |= u64(node) << 43; - } - - void SetChild(u32 index, u32 node) { - const u32 shift = (index ? 22 : 1); - header &= ~(NodeIndexMask << shift); - header |= u64(node) << shift; - } - - u32 Parent() const { - return (header >> 43) & NodeIndexMask; - } - - u32 Child(u32 index) const { - const u32 shift = (index ? 22 : 1); - return (header >> shift) & NodeIndexMask; - } - - bool IsRoot() const { - return Parent() == 0u; - } -}; - -/** - * Provides a two-part hash table for read and written resource ranges, which is - * backed by binary trees to handle individual address ranges as well as collisions. - */ -class BarrierTracker { - constexpr static u32 HashTableSize = 32u; - -public: - explicit BarrierTracker(); - ~BarrierTracker() = default; - - /// Checks whether there is a pending access of a given type - bool FindRange(const AddressRange& range, Access access_type) const; - - /// Inserts address range for a given access type - void InsertRange(const AddressRange& range, Access access_type); - - /// Clears the entire structure - void Clear(); - - /// Checks whether any resources are dirty - bool Empty() const { - return !root_mask_valid; - } - -private: - u32 AllocateNode(); - void FreeNode(u32 node); - u32 FindNode(const AddressRange& range, u32 root_index) const; - u32 InsertNode(const AddressRange& range, u32 root_index); - void RemoveNode(u32 node_index, u32 root_index); - void RebalancePostInsert(u32 node_index, u32 root_index); - void RotateLeft(u32 node_index, u32 root_index); - void RotateRight(u32 node_index, u32 root_index); - - constexpr static u32 ComputeRootIndex(const AddressRange& range, Access access) { - u64 hash = range.resource * 93887; - hash ^= (hash >> 16); - return 1 + (hash % HashTableSize) + (access == Access::Write ? HashTableSize : 0); - } - -private: - u64 root_mask_valid{}; - u64 root_mask_subtree{}; - std::vector nodes; - std::vector free; -}; - -} // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp index afcb5978c..307ff5ca1 100644 --- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp +++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp @@ -32,10 +32,13 @@ GraphicsPipeline::GraphicsPipeline( const Shader::Profile& profile, const GraphicsPipelineKey& key_, vk::PipelineCache pipeline_cache, std::span infos, std::span runtime_infos, - std::optional fetch_shader_, - std::span modules, SerializationSupport& sdata, bool preloading) - : Pipeline{instance, scheduler, desc_heap, profile, pipeline_cache}, key{key_}, - fetch_shader{std::move(fetch_shader_)} { + const Shader::Gcn::FetchShaderData* fetch_shader_, std::span modules, + SerializationSupport& sdata, bool preloading) + : Pipeline{instance, scheduler, desc_heap, profile, pipeline_cache}, key{key_} { + if (fetch_shader_) { + fetch_shader = *fetch_shader_; + } + const vk::Device device = instance.GetDevice(); std::ranges::copy(infos, stages.begin()); BuildDescSetLayout(preloading); @@ -436,21 +439,22 @@ void GraphicsPipeline::GetVertexInputs( VertexInputs& divisors, VertexInputs& guest_buffers, u32 step_rate_0, u32 step_rate_1) const { using InstanceIdType = Shader::Gcn::VertexAttribute::InstanceIdType; - if (!fetch_shader || fetch_shader->attributes.empty()) { + if (fetch_shader.Empty() || fetch_shader.attributes.empty()) { return; } const auto& vs_info = GetStage(Shader::SwStage::Vertex); - for (const auto& attrib : fetch_shader->attributes) { + for (u32 semantic = 0; semantic < fetch_shader.attributes.size(); ++semantic) { + const auto& attrib = fetch_shader.attributes[semantic]; const auto step_rate = attrib.GetStepRate(); const auto buffer = attrib.GetSharp(vs_info); attributes.push_back(Attribute{ - .location = attrib.semantic, - .binding = attrib.semantic, + .location = semantic, + .binding = semantic, .format = LiverpoolToVK::SurfaceFormat(buffer.GetDataFmt(), buffer.GetNumberFmt()), .offset = 0, }); bindings.push_back(Binding{ - .binding = attrib.semantic, + .binding = semantic, .stride = buffer.GetStride(), .inputRate = step_rate == InstanceIdType::None ? vk::VertexInputRate::eVertex : vk::VertexInputRate::eInstance, @@ -462,7 +466,7 @@ void GraphicsPipeline::GetVertexInputs( bindings.back().divisor = divisor; } else if (step_rate != InstanceIdType::None) { divisors.push_back(vk::VertexInputBindingDivisorDescriptionEXT{ - .binding = attrib.semantic, + .binding = semantic, .divisor = divisor, }); } diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.h b/src/video_core/renderer_vulkan/vk_graphics_pipeline.h index 2f337897c..10acda30e 100644 --- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.h +++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.h @@ -88,12 +88,12 @@ public: vk::PipelineCache pipeline_cache, std::span stages, std::span runtime_infos, - std::optional fetch_shader, + const Shader::Gcn::FetchShaderData* fetch_shader, std::span modules, SerializationSupport& sdata, bool preloading); ~GraphicsPipeline(); - const std::optional& GetFetchShader() const noexcept { + const Shader::Gcn::FetchShaderData& GetFetchShader() const noexcept { return fetch_shader; } @@ -113,7 +113,7 @@ private: private: GraphicsPipelineKey key; - std::optional fetch_shader{}; + Shader::Gcn::FetchShaderData fetch_shader{}; }; struct ClipDistanceShaderKey { diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp index 50de9eef3..29feefbb3 100644 --- a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp @@ -356,7 +356,6 @@ const GraphicsPipeline* PipelineCache::GetGraphicsPipeline(const DrawIndirectPar } } } - fetch_shader.reset(); } return it->second.get(); } @@ -489,7 +488,7 @@ bool PipelineCache::RefreshGraphicsKey() { bool PipelineCache::RefreshGraphicsStages() { const auto& regs = liverpool->regs; auto& key = graphics_key; - fetch_shader = std::nullopt; + fetch_shader = nullptr; Shader::Backend::Bindings binding{}; const auto bind_stage = [&](HwStage stage_in, SwStage stage_out) -> bool { @@ -509,13 +508,8 @@ bool PipelineCache::RefreshGraphicsStages() { } const auto params = AmdGpu::GetParams(*pgm); - std::optional fetch_shader_; - std::tie(infos[stage_out_idx], modules[stage_out_idx], fetch_shader_, - key.stage_hashes[stage_out_idx]) = + std::tie(infos[stage_out_idx], modules[stage_out_idx], key.stage_hashes[stage_out_idx]) = GetProgram(stage_in, stage_out, params, binding); - if (fetch_shader_) { - fetch_shader = fetch_shader_; - } return true; }; @@ -614,7 +608,7 @@ bool PipelineCache::RefreshComputeKey() { Shader::Backend::Bindings binding{}; const auto& cs_pgm = liverpool->GetCsRegs(); const auto cs_params = AmdGpu::GetParams(cs_pgm); - std::tie(infos[0], modules[0], fetch_shader, compute_key.value) = + std::tie(infos[0], modules[0], compute_key.value) = GetProgram(HwStage::Compute, SwStage::Compute, cs_params, binding); return true; } @@ -667,8 +661,10 @@ PipelineCache::Result PipelineCache::GetProgram(HwStage hw_stage, SwStage sw_sta RegisterShaderMeta(program->info, spec.fetch_shader_data, spec, perm_hash, 0); program->AddPermut(module, std::move(spec)); - return std::make_tuple(&program->info, module, program->modules[0].spec.fetch_shader_data, - perm_hash); + if (auto& fetch = program->modules[0].spec.fetch_shader_data; !fetch.Empty()) { + fetch_shader = &fetch; + } + return std::make_tuple(&program->info, module, perm_hash); } auto& program = it_pgm.value(); @@ -696,8 +692,10 @@ PipelineCache::Result PipelineCache::GetProgram(HwStage hw_stage, SwStage sw_sta perm_idx = std::distance(program->modules.begin(), it); perm_hash = HashCombine(params.hash, perm_idx); } - return std::make_tuple(&program->info, module, - program->modules[perm_idx].spec.fetch_shader_data, perm_hash); + if (auto& fetch = program->modules[perm_idx].spec.fetch_shader_data; !fetch.Empty()) { + fetch_shader = &fetch; + } + return std::make_tuple(&program->info, module, perm_hash); } std::optional PipelineCache::ReplaceShader(vk::ShaderModule module, diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.h b/src/video_core/renderer_vulkan/vk_pipeline_cache.h index da18ba4d5..edd9737b8 100644 --- a/src/video_core/renderer_vulkan/vk_pipeline_cache.h +++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.h @@ -11,6 +11,7 @@ #include "video_core/renderer_vulkan/vk_compute_pipeline.h" #include "video_core/renderer_vulkan/vk_graphics_pipeline.h" #include "video_core/renderer_vulkan/vk_resource_pool.h" +#include "vulkan/vulkan.hpp" template <> struct std::hash { @@ -85,9 +86,8 @@ public: const ComputePipeline* GetComputePipeline(); - using Result = std::tuple, u64>; - Result GetProgram(Shader::HwStage stage, Shader::SwStage l_stage, + using Result = std::tuple; + Result GetProgram(Shader::HwStage hw_stage, Shader::SwStage sw_stage, const Shader::ShaderParams& params, Shader::Backend::Bindings& binding); std::optional ReplaceShader(vk::ShaderModule module, @@ -134,7 +134,7 @@ private: std::array runtime_infos{}; std::array infos{}; std::array modules{}; - std::optional fetch_shader{}; + Shader::Gcn::FetchShaderData* fetch_shader{}; GraphicsPipelineKey graphics_key{}; ComputePipelineKey compute_key{}; u32 num_new_pipelines{}; // new pipelines added to the cache since the game start diff --git a/src/video_core/renderer_vulkan/vk_pipeline_serialization.cpp b/src/video_core/renderer_vulkan/vk_pipeline_serialization.cpp index 2c6fc3c3c..f755c4b4e 100644 --- a/src/video_core/renderer_vulkan/vk_pipeline_serialization.cpp +++ b/src/video_core/renderer_vulkan/vk_pipeline_serialization.cpp @@ -92,7 +92,6 @@ void RegisterShaderBinary(std::vector&& spv, u64 pgm_hash, size_t perm_idx) } bool LoadShaderMeta(Serialization::Archive& ar, Shader::Info& info, - std::optional& fetch_shader_data, Shader::StageSpecialization& spec, size_t& perm_idx) { Serialization::Reader meta{ar}; @@ -114,8 +113,6 @@ bool LoadShaderMeta(Serialization::Archive& ar, Shader::Info& info, spec.Deserialize(ar); info.Deserialize(ar); - - fetch_shader_data = spec.fetch_shader_data; return true; } @@ -243,7 +240,7 @@ bool PipelineCache::LoadGraphicsPipeline(Serialization::Archive& ar) { infos.fill(nullptr); modules.fill(nullptr); - fetch_shader.reset(); + fetch_shader = nullptr; return true; } @@ -253,7 +250,7 @@ bool PipelineCache::LoadPipelineStage(Serialization::Archive& ar, size_t stage) Shader::StageSpecialization spec{}; spec.info = &program->info; size_t perm_idx{}; - if (!LoadShaderMeta(ar, program->info, fetch_shader, spec, perm_idx)) { + if (!LoadShaderMeta(ar, program->info, spec, perm_idx)) { return false; } @@ -297,6 +294,9 @@ bool PipelineCache::LoadPipelineStage(Serialization::Archive& ar, size_t stage) infos[stage] = &it_pgm.value()->info; modules[stage] = module; + if (auto& fetch = it_pgm.value()->modules[perm_idx].spec.fetch_shader_data; !fetch.Empty()) { + fetch_shader = &fetch; + } return true; } @@ -453,9 +453,9 @@ void StageSpecialization::Serialize(Serialization::Archive& ar) const { spec.Write(bitset.to_string()); - if (fetch_shader_data) { - spec.Write(sizeof(*fetch_shader_data)); - fetch_shader_data->Serialize(ar); + if (!fetch_shader_data.Empty()) { + spec.Write(sizeof(fetch_shader_data)); + fetch_shader_data.Serialize(ar); } else { spec.Write(size_t{0}); } diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index 2499693b0..2663a8769 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -143,17 +143,16 @@ void Rasterizer::PrepareRenderState(const GraphicsPipeline* pipeline) { } } -static std::pair GetDrawOffsets( - const AmdGpu::Regs& regs, const Shader::Info& info, - const std::optional& fetch_shader) { +static std::pair GetDrawOffsets(const AmdGpu::Regs& regs, const Shader::Info& info, + const Shader::Gcn::FetchShaderData& fetch_shader) { u32 vertex_offset = regs.index_offset; u32 instance_offset = 0; - if (fetch_shader) { - if (vertex_offset == 0 && fetch_shader->vertex_offset_sgpr != -1) { - vertex_offset = info.user_data[fetch_shader->vertex_offset_sgpr]; + if (!fetch_shader.Empty()) { + if (vertex_offset == 0 && fetch_shader.vertex_offset_sgpr != -1) { + vertex_offset = info.user_data[fetch_shader.vertex_offset_sgpr]; } - if (fetch_shader->instance_offset_sgpr != -1) { - instance_offset = info.user_data[fetch_shader->instance_offset_sgpr]; + if (fetch_shader.instance_offset_sgpr != -1) { + instance_offset = info.user_data[fetch_shader.instance_offset_sgpr]; } } return {vertex_offset, instance_offset}; @@ -729,7 +728,7 @@ void Rasterizer::BindBuffers(const Shader::Info& stage, Shader::Backend::Binding if (desc.buffer_type == Shader::BufferType::GdsBuffer) { const auto* gds_buf = buffer_cache.GetGdsBuffer(); buffer_infos.emplace_back(gds_buf->Handle(), 0, gds_buf->SizeBytes()); - needs_barrier |= runtime.IsBufferAccessed(gds_buf, 0, gds_buf->SizeBytes()); + bound_buffers.emplace_back(gds_buf, 0, gds_buf->SizeBytes(), desc.is_written); } else if (desc.buffer_type == Shader::BufferType::Flatbuf) { auto& vk_buffer = buffer_cache.GetStreamBuffer(); const u32 ubo_size = stage.flattened_ud_buf.size() * sizeof(u32); @@ -908,7 +907,8 @@ void Rasterizer::BindTextures(const Shader::Info& stage, Shader::Backend::Bindin } else { needs_barrier |= runtime.Transit( &image, vk::ImageLayout::eGeneral, vk::PipelineStageFlagBits2::eAllCommands, - vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eShaderWrite); + vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eShaderWrite, + desc.view_info.range); } } else { if (is_storage) { @@ -1166,7 +1166,8 @@ void Rasterizer::FillBuffer(VAddr address, u32 num_bytes, u32 value, bool is_gds } void Rasterizer::CopyBuffer(VAddr dst, VAddr src, u32 num_bytes, bool dst_gds, bool src_gds) { - if (!dst_gds && !buffer_cache.IsRegionGpuModified(dst, num_bytes)) { + if (!dst_gds && !buffer_cache.IsRegionGpuModified(dst, num_bytes) && + !buffer_cache.IsRegionInSyncBatch(dst, num_bytes)) { if (!src_gds && !buffer_cache.IsRegionGpuModified(src, num_bytes) && !texture_cache.FindImageFromRange(src, num_bytes)) { // Both buffers were not transferred to GPU yet. Can safely copy in host memory. diff --git a/src/video_core/renderer_vulkan/vk_runtime.cpp b/src/video_core/renderer_vulkan/vk_runtime.cpp index c5a606777..ca5446a7d 100644 --- a/src/video_core/renderer_vulkan/vk_runtime.cpp +++ b/src/video_core/renderer_vulkan/vk_runtime.cpp @@ -679,24 +679,21 @@ void Runtime::SetBackingSamples(VideoCore::Image* image, u32 num_samples, bool c bool Runtime::IsBufferAccessed(const VideoCore::Buffer* handle, u64 offset, u64 size, bool check_read_access) { - const AddressRange range = { - .resource = reinterpret_cast(handle), - .range_start = offset, - .range_end = offset + size - 1, - }; - bool has_access = barrier_tracker.FindRange(range, Access::Write); + MakeCurrent(handle); + bool has_access = resource->write_ranges.Overlaps(offset, offset + size); if (check_read_access && !has_access) { - has_access |= barrier_tracker.FindRange(range, Access::Read); + has_access |= resource->read_ranges.Overlaps(offset, offset + size); } return has_access; } void Runtime::AccessBuffer(const VideoCore::Buffer* handle, u64 offset, u64 size, vk::PipelineStageFlags2 src_stage, vk::AccessFlags2 src_access) { - const AddressRange range = { - .resource = reinterpret_cast(handle), - .range_start = offset, - .range_end = offset + size - 1, + MakeCurrent(handle); + + const Interval range = { + .start = offset, + .end = offset + size, }; constexpr static vk::AccessFlags2 READ_MASK = @@ -712,10 +709,10 @@ void Runtime::AccessBuffer(const VideoCore::Buffer* handle, u64 offset, u64 size vk::AccessFlagBits2::eMemoryWrite | vk::AccessFlagBits2::eTransformFeedbackWriteEXT; if (src_access & WRITE_MASK) { - barrier_tracker.InsertRange(range, Access::Write); + resource->write_ranges.Add(range); } if (src_access & READ_MASK) { - barrier_tracker.InsertRange(range, Access::Read); + resource->read_ranges.Add(range); } memory_barrier.srcStageMask |= src_stage; @@ -746,7 +743,24 @@ void Runtime::FlushBarriers() { memory_barrier.srcAccessMask = vk::AccessFlagBits2::eNone; image_barriers.clear(); - barrier_tracker.Clear(); + for (auto& resource : resources) { + resource.read_ranges.Clear(); + resource.write_ranges.Clear(); + } + resources.clear(); + resource = nullptr; +} + +void Runtime::MakeCurrent(const VideoCore::Buffer* handle) { + if (resource && resource->handle == handle) { + return; + } + auto it = std::ranges::find(resources, handle, &BufferBarriers::handle); + if (it != resources.end()) { + resource = std::addressof(*it); + return; + } + resource = &resources.emplace_back(handle); } } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_runtime.h b/src/video_core/renderer_vulkan/vk_runtime.h index 6ac9f7b34..7dffced1d 100644 --- a/src/video_core/renderer_vulkan/vk_runtime.h +++ b/src/video_core/renderer_vulkan/vk_runtime.h @@ -3,9 +3,9 @@ #pragma once +#include "common/interval_set.h" #include "common/types.h" #include "video_core/buffer_cache/buffer.h" -#include "video_core/renderer_vulkan/vk_barrier_tracker.h" #include "video_core/renderer_vulkan/vk_staging_buffer_pool.h" #include "video_core/texture_cache/image.h" #include "video_core/texture_cache/types.h" @@ -76,12 +76,22 @@ public: void FlushBarriers(); +private: + void MakeCurrent(const VideoCore::Buffer* handle); + private: const Instance& instance; Scheduler& scheduler; std::unique_ptr blit_helper; StagingBufferPool staging_pool; - BarrierTracker barrier_tracker; + struct BufferBarriers { + const VideoCore::Buffer* handle; + using AccessList = IntervalList; + AccessList read_ranges; + AccessList write_ranges; + }; + BufferBarriers* resource{}; + std::vector resources; VideoCore::Image::Barriers image_barriers; vk::MemoryBarrier2 memory_barrier{}; }; diff --git a/src/video_core/texture_cache/image.cpp b/src/video_core/texture_cache/image.cpp index 5c80880c2..5706dd213 100644 --- a/src/video_core/texture_cache/image.cpp +++ b/src/video_core/texture_cache/image.cpp @@ -259,11 +259,7 @@ void Image::GetBarriers(Barriers& barriers, vk::ImageLayout dst_layout, vk::Acce ASSERT(subres_idx < subresource_states.size()); auto& state = subresource_states[subres_idx]; - constexpr auto write_flags = vk::AccessFlagBits2::eTransferWrite | - vk::AccessFlagBits2::eShaderWrite | - vk::AccessFlagBits2::eMemoryWrite; - const bool is_write = static_cast(state.access_mask & write_flags); - if (state.layout != dst_layout || state.access_mask != dst_mask || is_write) { + if (state.layout != dst_layout || state.access_mask != dst_mask) { barriers.emplace_back(vk::ImageMemoryBarrier2{ .srcStageMask = state.pl_stage, .srcAccessMask = state.access_mask, @@ -293,14 +289,9 @@ void Image::GetBarriers(Barriers& barriers, vk::ImageLayout dst_layout, vk::Acce subresource_states.clear(); } } else { // Full resource transition - constexpr auto write_flags = vk::AccessFlagBits2::eTransferWrite | - vk::AccessFlagBits2::eShaderWrite | - vk::AccessFlagBits2::eMemoryWrite; - const bool is_write = static_cast(last_state.access_mask & write_flags); - if (last_state.layout == dst_layout && last_state.access_mask == dst_mask && !is_write) { + if (last_state.layout == dst_layout && last_state.access_mask == dst_mask) { return; } - barriers.emplace_back(vk::ImageMemoryBarrier2{ .srcStageMask = last_state.pl_stage, .srcAccessMask = last_state.access_mask, diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 1984f1907..7884a8693 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -12,9 +12,9 @@ #include #include "common/lru_cache.h" +#include "common/multi_level_page_table.h" #include "common/slot_vector.h" #include "shader_recompiler/resource.h" -#include "video_core/multi_level_page_table.h" #include "video_core/texture_cache/blit_helper.h" #include "video_core/texture_cache/image.h" #include "video_core/texture_cache/image_view.h" @@ -49,7 +49,7 @@ class TextureCache { static constexpr size_t PAGE_BITS = 20; static constexpr bool NULL_CHECK = true; }; - using PageTable = MultiLevelPageTable; + using PageTable = Common::MultiLevelPageTable; public: enum class BindingType : u32 { @@ -119,7 +119,7 @@ public: /// Updates image contents if it was modified by CPU. void UpdateImage(ImageId image_id) { - std::scoped_lock lock{mutex}; + std::scoped_lock lk{mutex}; Image& image = slot_images[image_id]; TrackImage(image_id); TouchImage(image);