Fix(GPU): ported texture, GPU and shader cache fixes (#1124)

EN-US

- Fixed texture cache image references moving when the cache grows, out-of-range slot IDs, per-dimension extent checks, view clamping, padding-only overlaps and in-flight sampler destruction
- Fixed depth and colour buffer copies to copy every shared mip and layer
- Fixed scheduler null chunks at 0x20000, exit 23 from oversized recording data and stale idle signals releasing new submissions
- Fixed IRQ heap corruption at startup and saved packet sizes before running packets
- Fixed shader cache loss: queued writes drained at exit, partial loads rejected, damaged records skipped
- Fixed draws whose pipelines the driver refuses, speculative specialization exceptions and null user-data tables at area loads
- Updated texture pressure to start at 60% of the budget on cards below 8 GB, unblocked evictions behind retained images and kept GPU-only images below the critical mark
- Added tests for the slot vector, subresource extents, IRQ controller, record data, cache storage, cache flush, serdes and shader user data

---

PT-BR

- Corrigidos no cache de texturas: referências de imagem que mudavam de lugar quando o cache crescia, IDs de slot fora do intervalo, checagem de tamanho por dimensão, recorte de views, sobreposição só de padding e destruição de samplers em uso
- Corrigidas as cópias de buffers de depth e color para copiar todos os mips e layers compartilhados
- Corrigidos no scheduler: chunks nulos em 0x20000, exit 23 com dados de gravação grandes demais e sinais de ocioso antigos liberando submissões novas
- Corrigida a corrupção do heap de IRQ na inicialização, e o tamanho dos pacotes passou a ser salvo antes de executá-los
- Corrigida a perda do cache de shaders: gravações na fila esvaziadas ao sair, cargas parciais rejeitadas e registros danificados pulados
- Corrigidos os draws com pipeline recusado pelo driver, as exceções de especialização especulativa e as tabelas de user data nulas na troca de área
- Atualizada a pressão de texturas para começar em 60% do orçamento em placas com menos de 8 GB, desbloqueadas as remoções atrás de imagens retidas e mantidas as imagens só da GPU abaixo da marca crítica
- Adicionados testes de slot vector, subresource extents, controlador de IRQ, record data, cache storage, flush do cache, serdes e shader user data
This commit is contained in:
Mahdi Akbari Zarkesh authored and GitHub committed 2026-11-01 21:31:31 -03:00
1 parent e82ca703e4
commit c22329f13b
37 files changed
+1726 -202

No files matched your search

+8 -2
View File
@@ -34,10 +34,13 @@ MINGW*|MSYS*)
out/services-test.exe
clang -std=c11 -O2 -g -Wall -Wextra -Werror -pthread -I. -Isrc tests/test_savedata.c src/compat_win.c -lbcrypt -o out/savedata-test.exe
out/savedata-test.exe
ninja -C out/gpu graphics-settings-test player-hook-test ledge-hook-test
ninja -C out/gpu graphics-settings-test player-hook-test ledge-hook-test slot-vector-test subresource-extent-test irq-controller-test
out/gpu/graphics-settings-test.exe
out/gpu/player-hook-test.exe
out/gpu/ledge-hook-test.exe
out/gpu/slot-vector-test.exe
out/gpu/subresource-extent-test.exe
out/gpu/irq-controller-test.exe
fi
exit 0
;;
@@ -135,6 +138,9 @@ if [[ ${1:-} == --test ]]; then
"$CC" -std=c11 -O2 -g -Wall -Wextra -Werror -c src/isa_rewrite.c -I. -Isrc -o out/isa_rewrite.o
"$CC" -std=c11 -O2 -g -Wall -Wextra -Werror -mbmi -mlzcnt -I. -Isrc tests/test_isa_stub.c out/isa_rewrite.o out/isa_emulate.o -o out/isa-stub-test
out/isa-stub-test tests/data/isa_fixture.bin
ninja -C out/gpu graphics-settings-test
ninja -C out/gpu graphics-settings-test slot-vector-test subresource-extent-test irq-controller-test
out/gpu/graphics-settings-test
out/gpu/slot-vector-test
out/gpu/subresource-extent-test
out/gpu/irq-controller-test
fi
+44
View File
@@ -276,6 +276,42 @@ if (WIN32)
endif()
# Explicit test targets; neither launches the game nor needs a Vulkan device.
add_executable(slot-vector-test EXCLUDE_FROM_ALL ../tests/test_slot_vector.cpp
shim/common/slot_vector_pages.cpp)
target_compile_options(slot-vector-test PRIVATE -UNDEBUG -D_DEBUG -Wall -Wextra -Werror)
target_include_directories(slot-vector-test PRIVATE shim shadps4)
target_compile_definitions(slot-vector-test PRIVATE NOMINMAX WIN32_LEAN_AND_MEAN)
target_link_libraries(slot-vector-test PRIVATE fmt::fmt)
add_executable(subresource-extent-test EXCLUDE_FROM_ALL ../tests/test_subresource_extent.cpp)
target_compile_options(subresource-extent-test PRIVATE -UNDEBUG -Wall -Wextra -Werror)
target_include_directories(subresource-extent-test PRIVATE shim shadps4)
target_link_libraries(subresource-extent-test PRIVATE fmt::fmt)
find_package(Threads REQUIRED)
add_executable(irq-controller-test EXCLUDE_FROM_ALL ../tests/test_irq_controller.cpp
shim/common/assert.cpp shim/common/logging/log.cpp)
target_include_directories(irq-controller-test PRIVATE shim shadps4)
target_compile_options(irq-controller-test PRIVATE -UNDEBUG -Wall -Wextra -Werror)
target_link_libraries(irq-controller-test PRIVATE fmt::fmt magic_enum::magic_enum Threads::Threads)
add_executable(shader-user-data-test EXCLUDE_FROM_ALL ../tests/test_shader_user_data.cpp
shadps4/shader_recompiler/frontend/fetch_shader.cpp
shadps4/shader_recompiler/frontend/decode.cpp
shadps4/shader_recompiler/frontend/format.cpp)
target_compile_options(shader-user-data-test PRIVATE -UNDEBUG)
target_include_directories(shader-user-data-test PRIVATE shim shadps4 third_party/gcn/include)
target_link_libraries(shader-user-data-test PRIVATE Vulkan::Headers fmt::fmt Boost::headers magic_enum::magic_enum)
add_executable(cache-storage-test EXCLUDE_FROM_ALL ../tests/test_cache_storage.cpp
../tests/test_gpu_runtime_stubs.cpp)
target_include_directories(cache-storage-test PRIVATE
$<TARGET_PROPERTY:spartanportgpu,INCLUDE_DIRECTORIES>)
target_link_libraries(cache-storage-test PRIVATE spartanportgpu)
target_compile_options(cache-storage-test PRIVATE -UNDEBUG -fPIC)
if (WIN32)
target_sources(cache-storage-test PRIVATE ../src/compat_win.c)
target_compile_definitions(cache-storage-test PRIVATE sigjmp_buf=spartanport_jmp_buf)
target_compile_options(cache-storage-test PRIVATE
-include ${CMAKE_CURRENT_SOURCE_DIR}/../src/compat_win.h)
target_link_libraries(cache-storage-test PRIVATE onecore)
endif()
add_executable(pipeline-binary-cache-test EXCLUDE_FROM_ALL ../tests/test_pipeline_binary_cache.cpp
../tests/test_gpu_runtime_stubs.cpp)
target_compile_definitions(pipeline-binary-cache-test PRIVATE NOMINMAX WIN32_LEAN_AND_MEAN)
@@ -304,6 +340,10 @@ if (WIN32)
endif()
set_source_files_properties(../tests/test_pipeline_binary_cache.cpp PROPERTIES
COMPILE_OPTIONS "-fvisibility=hidden;-fvisibility-inlines-hidden")
add_executable(serdes-test EXCLUDE_FROM_ALL ../tests/test_serdes.cpp)
target_compile_options(serdes-test PRIVATE -UNDEBUG -Wall -Wextra -Werror)
target_include_directories(serdes-test PRIVATE shim shadps4)
target_link_libraries(serdes-test PRIVATE fmt::fmt)
add_executable(motion-history-test EXCLUDE_FROM_ALL ../tests/test_motion_history.cpp)
target_compile_options(motion-history-test PRIVATE -UNDEBUG -Wall -Wextra -Werror)
target_include_directories(motion-history-test PRIVATE ..)
@@ -345,6 +385,10 @@ target_compile_options(blend-rewrite-test PRIVATE -UNDEBUG)
target_include_directories(blend-rewrite-test PRIVATE shim shadps4 third_party/gcn/include)
target_link_libraries(blend-rewrite-test PRIVATE Vulkan::Headers fmt::fmt Boost::headers magic_enum::magic_enum)
add_executable(depth-stencil-state-test EXCLUDE_FROM_ALL ../tests/test_depth_stencil_state.cpp)
add_executable(record-data-test EXCLUDE_FROM_ALL ../tests/test_record_data.cpp)
target_compile_options(record-data-test PRIVATE -UNDEBUG)
target_include_directories(record-data-test PRIVATE shim shadps4 third_party/gcn/include)
target_link_libraries(record-data-test PRIVATE Vulkan::Headers fmt::fmt Boost::headers magic_enum::magic_enum)
add_executable(async-compiler-test EXCLUDE_FROM_ALL ../tests/test_async_compiler.cpp)
target_compile_options(async-compiler-test PRIVATE -UNDEBUG)
target_include_directories(async-compiler-test PRIVATE shadps4)
+8 -1
View File
@@ -3,8 +3,15 @@ Directories: src/video_core, src/shader_recompiler, src/common, src/core/librari
Also: src/core/libraries/kernel/{orbis_error,posix_error,time,process,equeue}.h, equeue.cpp, src/core/libraries/error_codes.h, src/core/platform.h, src/core/libraries/system/{userservice,systemservice}.h.
Also: src/core/libraries/avplayer (without the handle streamer), src/core/libraries/videodec/video_utils.{h,cpp}.
Local changes are marked with "spartanport:" comments.
Backported from upstream after b4e7ae7 (2026-10-02), recompiler correctness fixes only:
Backported from upstream after b4e7ae7 (2026-10-02), correctness fixes:
233562929 V_INTERP_MOV_F32 P10/P20 selectors (#5087), d96278e1e compute shader floating-point
mode (#5126), e518a6515 S_SUBB_U32 SCC wrap (#5031), 282e5c792 S_ADDC_U32 SCC carry-out (#5030),
fb6367d29 32-bit literals in the high dword of 64-bit floats (#5114), 94e217781 V_BFM_B32 (#5185),
d1cb90dbb ADDC (#5200), dd43d1cac V_CVT_PK_U8_F32 (#5194), ef08b1e1f V_ALIGNBIT/ALIGNBYTE (#5186).
0f6d1b870 stable SlotVector address-space reservation (#5212), adapted to the existing API.
359234bbb SlotVector allocation bitset bounds check (#5158).
ca36c7e31 subresource extents compared per dimension (#5524).
afde63182 buffer image copies between different layer counts (#5299).
b3d46e96f every shared mip level copied in CopyImageWithBuffer (#5368).
7ea677e56 packet sizes read before the packet runs (#5541).
2338a06f9 device-scaled texture GC floors only (#5193), via Julio Cacko c7d495ce3d.
+2 -1
View File
@@ -55,8 +55,9 @@ public:
template <typename Func>
void ForEachItemBelow(TickType tick, Func&& func) {
// spartanport: inspect the callback's result type so a spent budget stops the walk.
static constexpr bool RETURNS_BOOL =
std::is_same_v<std::invoke_result<Func, ObjectType>, bool>;
std::is_same_v<std::invoke_result_t<Func, ObjectType>, bool>;
Item* iterator = first_item;
while (iterator) {
if (static_cast<s64>(tick) - static_cast<s64>(iterator->tick) < 0) {
+29 -5
View File
@@ -7,10 +7,18 @@
#include "common/types.h"
#include <cstddef>
// spartanport: report damaged cache records through recoverable exceptions.
#include <stdexcept>
#include <vector>
namespace Serialization {
// spartanport: let cache loaders skip damaged records instead of stopping startup.
struct CorruptData : std::runtime_error {
CorruptData()
: std::runtime_error{"Invalid or corrupted deserialization container/shader cache"} {}
};
template <typename T>
concept Container = requires(T t) {
typename T::iterator;
@@ -41,9 +49,15 @@ struct Archive {
return container.data() + offset;
}
// spartanport: check against remaining bytes without overflowing offset + size.
[[nodiscard]] size_t Remaining() const {
return container.size() - offset;
}
void Advance(size_t size) {
ASSERT_MSG(offset + size <= container.size(),
"Invalid or corrupted deserialization container/shader cache");
if (size > Remaining()) {
throw CorruptData{};
}
offset += size;
}
@@ -105,8 +119,10 @@ struct Writer {
struct Reader {
template <typename T>
void Read(T* ptr, size_t size) {
ASSERT_MSG(ar.offset + size <= ar.container.size(),
"Invalid or corrupted deserialization container/shader cache");
// spartanport: reject truncated records before copying from them.
if (size > ar.Remaining()) {
throw CorruptData{};
}
std::memcpy(reinterpret_cast<void*>(ptr), ar.CurrPtr(), size);
ar.Advance(size);
}
@@ -121,7 +137,11 @@ struct Reader {
void Read(auto& v) {
size_t num_elements{};
Read(num_elements);
for (int i = 0; i < num_elements; ++i) {
// spartanport: every element needs a byte; reject corrupt counts before allocating.
if (num_elements > ar.Remaining()) {
throw CorruptData{};
}
for (size_t i = 0; i < num_elements; ++i) {
v.emplace_back();
Read(v.back());
}
@@ -130,6 +150,10 @@ struct Reader {
void Read(std::string& s) {
size_t length{};
Read(length);
// spartanport: reject corrupt string lengths before resizing the destination.
if (length > ar.Remaining()) {
throw CorruptData{};
}
s.resize(length);
Read(s.data(), length);
}
+52 -30
View File
@@ -3,7 +3,12 @@
#pragma once
#include <numeric>
// spartanport: include the standard declarations used by stable storage directly.
#include <cstdlib>
#include <functional>
#include <iterator>
#include <limits>
#include <new>
#include <utility>
#include <vector>
#include "common/assert.h"
@@ -26,10 +31,14 @@ struct SlotId {
u32 index = INVALID_INDEX;
};
// spartanport: reserve stable storage through the shim so inserts keep references valid.
void* SlotVectorReservePages(std::size_t size) noexcept;
void SlotVectorCommitPages(void* address, std::size_t size) noexcept;
void SlotVectorReleasePages(void* base, std::size_t size) noexcept;
// spartanport: grow by committed pages instead of a relocating initial allocation.
template <class T>
class SlotVector {
constexpr static std::size_t InitialCapacity = 2048;
public:
template <typename ValueType, typename Pointer, typename Reference>
class Iterator {
@@ -89,7 +98,8 @@ public:
using const_reverse_iterator = std::reverse_iterator<const_iterator>;
SlotVector() {
Reserve(InitialCapacity);
// spartanport: commit the first chunk inside a fixed address-space reservation.
AddChunk();
}
~SlotVector() noexcept {
@@ -102,9 +112,14 @@ public:
}
index += 64;
}
delete[] values;
// spartanport: release the reservation after destroying its live entries.
SlotVectorReleasePages(values, ReservedBytes);
}
// spartanport: the reservation has one owner and cannot be copied.
SlotVector(const SlotVector&) = delete;
SlotVector& operator=(const SlotVector&) = delete;
[[nodiscard]] T& operator[](SlotId id) noexcept {
ValidateIndex(id);
return values[id.index].object;
@@ -116,7 +131,8 @@ public:
}
bool is_allocated(SlotId id) const {
return ReadStorageBit(id.index);
// spartanport: reject out-of-range IDs before reading the allocation bitset.
return (id.index / 64) < stored_bitset.size() && ReadStorageBit(id.index);
}
template <typename... Args>
@@ -219,7 +235,8 @@ private:
[[nodiscard]] u32 FreeValueIndex() noexcept {
if (free_list.empty()) {
Reserve(values_capacity ? (values_capacity << 1) : 1);
// spartanport: commit more storage without moving existing entries.
AddChunk();
}
const u32 free_index = free_list.back();
@@ -227,37 +244,42 @@ private:
return free_index;
}
void Reserve(std::size_t new_capacity) noexcept {
Entry* const new_values = new Entry[new_capacity];
std::size_t index = 0;
for (u64 bits : stored_bitset) {
for (std::size_t bit = 0; bits; ++bit, bits >>= 1) {
const std::size_t i = index + bit;
if ((bits & 1) == 0) {
continue;
}
T& old_value = values[i].object;
new (&new_values[i].object) T(std::move(old_value));
old_value.~T();
}
index += 64;
// spartanport: commit chunks in the reserved range so image references never dangle on insert.
void AddChunk() noexcept {
if (!values) {
values = static_cast<Entry*>(SlotVectorReservePages(ReservedBytes));
}
const std::size_t old_capacity = values_capacity;
const std::size_t new_capacity = (committed_bytes + ChunkBytes) / sizeof(Entry);
if (committed_bytes > ReservedBytes - ChunkBytes) {
ASSERT_MSG(false, "SlotVector out of space");
std::abort();
}
SlotVectorCommitPages(reinterpret_cast<u8*>(values) + committed_bytes, ChunkBytes);
committed_bytes += ChunkBytes;
for (std::size_t i = old_capacity; i < new_capacity; ++i) {
new (&values[i]) Entry();
}
stored_bitset.resize((new_capacity + 63) / 64);
const std::size_t old_free_size = free_list.size();
free_list.resize(old_free_size + (new_capacity - values_capacity));
const std::size_t new_free_size = free_list.size();
std::iota(free_list.rbegin(), free_list.rbegin() + new_free_size - old_free_size,
static_cast<u32>(values_capacity));
delete[] values;
values = new_values;
// Lowest new index on top, as before.
for (std::size_t i = new_capacity; i-- > old_capacity;) {
free_list.push_back(static_cast<u32>(i));
}
values_capacity = new_capacity;
}
// spartanport: bound the reservation and require page-compatible entry size and alignment.
static constexpr std::size_t ChunkBytes = std::size_t{2} << 20; // 2 MiB committed at a time
static constexpr std::size_t ReservedBytes = std::size_t{1} << 30; // 1 GiB of address space
static_assert(sizeof(Entry) <= ChunkBytes);
// All supported hosts provide at least 4 KiB alignment for page reservations.
static_assert(alignof(Entry) <= 4096);
Entry* values = nullptr;
std::size_t values_capacity = 0;
// spartanport: track committed bytes independently of partially filled entries.
std::size_t committed_bytes = 0;
std::vector<u64> stored_bitset;
std::vector<u32> free_list;
@@ -79,8 +79,15 @@ static u32 asc_next_offs_dw[Liverpool::NumComputeRings];
static VAddr tessellation_factors_ring_addr = -1;
static constexpr u32 tessellation_offchip_buffer_size = 0x800000u;
// spartanport: remember which submissions the lock protects so an older idle cannot clear it.
static u64 lock_generation{};
static void ResetSubmissionLock(Platform::InterruptId irq) {
std::unique_lock lock{m_wait_idle};
// spartanport: keep later command buffers locked until their own GPU work retires.
if (submission_lock && liverpool->IdleGeneration() < lock_generation) {
return;
}
submission_lock = 0;
cv_lock.notify_all();
}
@@ -2360,8 +2367,13 @@ s32 PS4_SYSV_ABI sceGnmSubmitDone() {
LOG_DEBUG(Lib_GnmDriver, "called");
std::scoped_lock lk{m_submit_lock};
WaitGpuIdle();
if (!liverpool->IsGpuIdle()) {
submission_lock = true;
// spartanport: serialize lock setup with idle handling and record the protected generation.
{
std::unique_lock lock{m_wait_idle};
if (!liverpool->IsGpuIdle()) {
submission_lock = true;
lock_generation = liverpool->SubmissionsTotal();
}
}
liverpool->SubmitDone();
send_init_packet = true;
+12 -5
View File
@@ -10,6 +10,8 @@
#include <magic_enum/magic_enum.hpp>
// spartanport: fixed IRQ storage avoids concurrent map growth.
#include <array>
#include <functional>
#include <mutex>
#include <unordered_map>
@@ -38,7 +40,8 @@ struct IrqController {
void RegisterOnce(InterruptId irq, IrqHandler handler) {
ASSERT_MSG(static_cast<u32>(irq) <= static_cast<u32>(InterruptId::InterruptIdMax),
"Invalid IRQ number");
auto& ctx = irq_contexts.try_emplace(irq).first->second;
// spartanport: index preallocated contexts so concurrent first use never inserts.
auto& ctx = irq_contexts[static_cast<u32>(irq)];
std::unique_lock lock{ctx.m_lock};
ctx.one_time_subscribers.emplace(handler);
}
@@ -46,7 +49,8 @@ struct IrqController {
void Register(InterruptId irq, IrqHandler handler, void* uid) {
ASSERT_MSG(static_cast<u32>(irq) <= static_cast<u32>(InterruptId::InterruptIdMax),
"Invalid IRQ number");
auto& ctx = irq_contexts.try_emplace(irq).first->second;
// spartanport: index preallocated contexts so concurrent first use never inserts.
auto& ctx = irq_contexts[static_cast<u32>(irq)];
std::unique_lock lock{ctx.m_lock};
ASSERT_MSG(ctx.persistent_handlers.find(uid) == ctx.persistent_handlers.cend(),
@@ -57,7 +61,8 @@ struct IrqController {
void Unregister(InterruptId irq, void* uid) {
ASSERT_MSG(static_cast<u32>(irq) <= static_cast<u32>(InterruptId::InterruptIdMax),
"Invalid IRQ number");
auto& ctx = irq_contexts.try_emplace(irq).first->second;
// spartanport: index preallocated contexts so concurrent first use never inserts.
auto& ctx = irq_contexts[static_cast<u32>(irq)];
std::unique_lock lock{ctx.m_lock};
ctx.persistent_handlers.erase(uid);
}
@@ -65,7 +70,8 @@ struct IrqController {
void Signal(InterruptId irq) {
ASSERT_MSG(static_cast<u32>(irq) <= static_cast<u32>(InterruptId::InterruptIdMax),
"Unexpected IRQ signaled");
auto& ctx = irq_contexts.try_emplace(irq).first->second;
// spartanport: index preallocated contexts so concurrent first use never inserts.
auto& ctx = irq_contexts[static_cast<u32>(irq)];
std::unique_lock lock{ctx.m_lock};
LOG_TRACE(Core, "IRQ signaled: {}", magic_enum::enum_name(irq));
@@ -88,7 +94,8 @@ private:
std::queue<IrqHandler> one_time_subscribers{};
std::mutex m_lock{};
};
std::unordered_map<InterruptId, IrqContext> irq_contexts{};
// spartanport: create every context up front to prevent registration/signal heap races.
std::array<IrqContext, static_cast<u32>(InterruptId::InterruptIdMax) + 1> irq_contexts{};
};
using IrqC = Common::Singleton<IrqController>;
+25 -1
View File
@@ -207,14 +207,38 @@ struct Info : InfoPersistent {
return *reinterpret_cast<const T*>(&FlatUserData()[sharp_idx]);
}
// spartanport: missing pointer pairs and low table addresses leave no vertex input to fetch.
bool UdPointerNull(u32 ptr_index) const noexcept {
if (ptr_index == IR::NumScalarRegs) {
return false;
}
const auto ud = UserData();
if (ptr_index >= ud.size() || ud.size() - ptr_index < 2) {
return true;
}
u64 pointer;
std::memcpy(&pointer, &ud[ptr_index], sizeof(pointer));
return (pointer & 0xFFFFFFFFFFFFULL) < 0x10000;
}
template <typename T>
T ReadUdReg(u32 ptr_index, u32 dword_offset) const noexcept {
T data;
// spartanport: null or short user data reads as an empty descriptor instead of faulting.
T data{};
const auto ud = UserData();
const u32* base = ud.data();
if (ptr_index != IR::NumScalarRegs) {
if (ptr_index >= ud.size() || ud.size() - ptr_index < 2) {
return data;
}
std::memcpy(&base, &ud[ptr_index], sizeof(base));
base = reinterpret_cast<const u32*>(VAddr(base) & 0xFFFFFFFFFFFFULL);
if (VAddr(base) < 0x10000) {
return data;
}
} else if (dword_offset > ud.size() ||
sizeof(T) > (ud.size() - dword_offset) * sizeof(u32)) {
return data;
}
std::memcpy(&data, base + dword_offset, sizeof(T));
return data;
+40 -18
View File
@@ -86,8 +86,14 @@ Liverpool::~Liverpool() {
}
void Liverpool::ProcessCommands() {
if (num_commands && rasterizer) {
rasterizer->DrainDrawPipe(Vulkan::DrawPipe::ReasonCommands); // spartanport: commands touch the caches
if (num_commands == 0) {
return;
}
// spartanport: commands touch the caches and the scheduler. No command may run before the
// drain: one queued by a guest thread after a check that saw none would run beside the draw
// recording thread (both recording into the scheduler: null record chunk).
if (rasterizer) {
rasterizer->DrainDrawPipe(Vulkan::DrawPipe::ReasonCommands);
}
// Process incoming commands with high priority
while (num_commands) {
@@ -158,8 +164,9 @@ void Liverpool::Process(std::stop_token stoken) {
std::scoped_lock lock{queue.m_access};
queue.submits.pop();
--num_submits;
// spartanport: retire the task count under the publication lock, queue then submit.
std::scoped_lock lock2{submit_mutex};
--num_submits;
submit_cv.notify_all();
}
}
@@ -184,13 +191,22 @@ void Liverpool::Process(std::stop_token stoken) {
rasterizer->DrainDrawPipe(Vulkan::DrawPipe::ReasonSubmissionEnd);
rasterizer->WaitDeferredSignals();
}
// spartanport: new submissions during the drain must finish before idle releases their
// command buffers; the generation also protects against a later submission lock.
bool idle;
u64 generation;
{
std::scoped_lock lk{submit_mutex};
if (num_submits == 0) {
idle = num_submits == 0;
generation = submissions_total;
if (idle) {
work_retired = true;
}
}
Platform::IrqC::Instance()->Signal(Platform::InterruptId::GpuIdle);
if (idle) {
idle_generation.store(generation, std::memory_order_release);
Platform::IrqC::Instance()->Signal(Platform::InterruptId::GpuIdle);
}
}
}
@@ -414,6 +430,8 @@ Liverpool::Task Liverpool::ProcessCeUpdate(std::span<const u32> ccb) {
}
const PM4ItOpcode opcode = header->type3.opcode;
// spartanport: save the size before execution can let the guest reuse this buffer.
const u32 packet_words = header->type3.NumWords() + 1;
const auto* it_body = reinterpret_cast<const u32*>(header) + 1;
switch (opcode) {
case PM4ItOpcode::Nop: {
@@ -464,7 +482,8 @@ Liverpool::Task Liverpool::ProcessCeUpdate(std::span<const u32> ccb) {
UNREACHABLE_MSG("Unknown PM4 type 3 opcode {:#x} with count {}",
static_cast<u32>(opcode), count);
}
ccb = NextPacket(ccb, header->type3.NumWords() + 1);
// spartanport: advance by the saved size because execution may invalidate the header.
ccb = NextPacket(ccb, packet_words);
}
FIBER_EXIT;
@@ -1308,8 +1327,8 @@ Liverpool::Task Liverpool::ProcessGraphics(std::span<const u32> dcb, std::span<c
}
const auto skip = *cond_exec->Address() == false;
if (skip) {
dcb = NextPacket(dcb,
header->type3.NumWords() + 1 + cond_exec->exec_count.Value());
// spartanport: use the size saved before execution when skipping packets too.
dcb = NextPacket(dcb, count + 1 + cond_exec->exec_count.Value());
continue;
}
break;
@@ -1318,7 +1337,8 @@ Liverpool::Task Liverpool::ProcessGraphics(std::span<const u32> dcb, std::span<c
UNREACHABLE_MSG("Unknown PM4 type 3 opcode {:#x} with count {}",
static_cast<u32>(opcode), count);
}
dcb = NextPacket(dcb, header->type3.NumWords() + 1);
// spartanport: use the saved size; flip or EOP signals let the guest reuse this buffer.
dcb = NextPacket(dcb, count + 1);
break;
}
}
@@ -1694,11 +1714,12 @@ void Liverpool::SubmitGfx(std::span<const u32> dcb, std::span<const u32> ccb) {
}
auto task = ProcessGraphics(dcb, ccb, seq);
queue.submits.emplace(task.handle);
// spartanport: publish the task and its counts together, locking queue then submit.
std::scoped_lock lk{submit_mutex};
++num_submits;
++submissions_total;
work_retired = false;
}
std::scoped_lock lk{submit_mutex};
++num_submits;
work_retired = false;
submit_cv.notify_one();
}
@@ -1711,12 +1732,13 @@ void Liverpool::SubmitAsc(u32 gnm_vqid, std::span<const u32> acb) {
{
std::scoped_lock lock{queue.m_access};
queue.submits.emplace(task.handle);
// spartanport: publish compute work and its counts together, locking queue then submit.
std::scoped_lock lk{submit_mutex};
num_mapped_queues = std::max(num_mapped_queues, gnm_vqid + 1);
++num_submits;
++submissions_total;
work_retired = false;
}
std::scoped_lock lk{submit_mutex};
num_mapped_queues = std::max(num_mapped_queues, gnm_vqid + 1);
++num_submits;
work_retired = false;
submit_cv.notify_one();
}
+14
View File
@@ -122,6 +122,17 @@ public:
return num_submits == 0 && (work_retired || !use_retired);
}
// spartanport: record the submission count so an older idle cannot release a newer lock.
u64 SubmissionsTotal() {
std::scoped_lock lk{submit_mutex};
return submissions_total;
}
// spartanport: read the completed count so idle handlers can reject stale interrupts.
u64 IdleGeneration() const {
return idle_generation.load(std::memory_order_acquire);
}
void SetVoPort(Libraries::VideoOut::VideoOutPort* port) {
vo_port = port;
}
@@ -271,6 +282,9 @@ private:
/// decrements num_submits once it has decoded a submission, before that). Under
/// submit_mutex with num_submits.
std::atomic<bool> work_retired{true};
// spartanport: distinguish submitted work from the generation covered by the idle interrupt.
std::atomic<u64> submissions_total{};
std::atomic<u64> idle_generation{};
std::atomic<u32> num_commands{};
std::atomic<bool> submit_done{};
std::mutex submit_mutex;
+58 -38
View File
@@ -14,17 +14,19 @@
#include <miniz.h>
#include <condition_variable>
// spartanport: callbacks report write exceptions instead of hiding them in unused futures.
#include <exception>
#include <functional>
#include <future>
#include <mutex>
#include <queue>
namespace {
std::mutex submit_mutex{};
u32 num_requests{};
// spartanport: one mutex guards queued and active writes, avoiding a racing request count.
std::condition_variable_any request_cv{};
std::queue<std::packaged_task<void()>> req_queue{};
std::condition_variable idle_cv{};
std::queue<std::function<void()>> req_queue{};
bool io_busy{};
std::mutex m_request{};
mz_zip_archive zip_ar{};
@@ -37,33 +39,25 @@ namespace Storage {
void ProcessIO(const std::stop_token& stoken) {
Common::SetCurrentThreadName("shadPS4:PipelineCacheIO");
while (!stoken.stop_requested()) {
// spartanport: a stop drains accepted writes; idle includes the callback currently running.
for (;;) {
std::function<void()> request;
{
std::unique_lock lk{submit_mutex};
Common::CondvarWait(request_cv, lk, stoken, [&] { return num_requests; });
}
if (stoken.stop_requested()) {
break;
}
while (num_requests) {
std::packaged_task<void()> request{};
{
std::scoped_lock lock{m_request};
if (req_queue.empty()) {
continue;
}
request = std::move(req_queue.front());
req_queue.pop();
std::unique_lock lk{m_request};
io_busy = false;
idle_cv.notify_all();
Common::CondvarWait(request_cv, lk, stoken, [&] { return !req_queue.empty(); });
if (req_queue.empty()) {
return;
}
if (request.valid()) {
request();
request.get_future().wait();
}
--num_requests;
request = std::move(req_queue.front());
req_queue.pop();
io_busy = true;
}
try {
request();
} catch (const std::exception& e) {
LOG_ERROR(Render, "Cache write failed: {}", e.what());
}
}
}
@@ -111,8 +105,13 @@ void DataBase::Open() {
}
} else {
cache_path = GetUserPath(PathType::CacheDir) / game_info.GameSerial();
if (!std::filesystem::exists(cache_path)) {
std::filesystem::create_directories(cache_path);
// spartanport: disable caching on directory errors so startup does not throw.
std::error_code error;
std::filesystem::create_directories(cache_path, error);
if (error) {
LOG_WARNING(Render, "Shader cache unavailable at {}: {}", cache_path.string(),
error.message());
return;
}
}
@@ -137,10 +136,20 @@ void DataBase::Close() {
LOG_INFO(Render, "Cache dumped");
}
// spartanport: bound exit waits without joining a worker that may be blocked on disk IO.
bool DataBase::Flush(std::chrono::milliseconds timeout) {
if (!IsOpened()) {
return true;
}
std::unique_lock lk{m_request};
return idle_cv.wait_for(lk, timeout, [] { return req_queue.empty() && !io_busy; });
}
template <typename T>
bool WriteVector(const BlobType type, std::filesystem::path&& path_, std::vector<T>&& v) {
{
auto request = std::packaged_task<void()>{[=]() {
// spartanport: move the payload into the queue to avoid copying it on the GPU thread.
std::function<void()> request = [type, path_ = std::move(path_), v = std::move(v)]() {
auto path{path_};
path.replace_extension(GetBlobFileExtension(type));
if (EmulatorSettings.IsPipelineCacheArchived()) {
@@ -152,7 +161,7 @@ bool WriteVector(const BlobType type, std::filesystem::path&& path_, std::vector
}
} else {
// spartanport: write beside and rename into place, so a game that ends during the
// write (the queue is not drained at exit) never leaves a truncated blob that
// write (an exit flush may time out) never leaves a truncated blob that
// the next start would load.
using namespace Common::FS;
auto temp{path};
@@ -166,13 +175,12 @@ bool WriteVector(const BlobType type, std::filesystem::path&& path_, std::vector
std::error_code error;
std::filesystem::rename(temp, path, error);
}
}};
};
std::scoped_lock lock{m_request};
req_queue.emplace(std::move(request));
}
std::scoped_lock lk{submit_mutex};
++num_requests;
// spartanport: queue insertion and the worker predicate now share m_request.
request_cv.notify_one();
return true;
}
@@ -193,9 +201,21 @@ void LoadVector(BlobType type, std::filesystem::path& path, std::vector<T>& v) {
v.resize(stat.m_uncomp_size / sizeof(T));
mz_zip_reader_extract_to_mem(&zip_ar, index, v.data(), stat.m_uncomp_size, 0);
} else {
// spartanport: clear failed loads so partial words and short reads remain cache misses.
v.clear();
const auto file = IOFile{path, FileAccessMode::Read};
v.resize(file.GetSize() / sizeof(T));
file.Read(v);
if (!file.IsOpen()) {
return;
}
const auto bytes = file.GetSize();
if (bytes % sizeof(T) != 0) {
return;
}
v.resize(bytes / sizeof(T));
if (file.Read(v) != v.size()) {
v.clear();
return;
}
}
}
+10 -2
View File
@@ -7,6 +7,9 @@
#include "common/singleton.h"
#include "common/types.h"
// spartanport: exit hooks observe startup safely and bound cache-drain waits.
#include <atomic>
#include <chrono>
#include <functional>
#include <thread>
#include <vector>
@@ -23,11 +26,15 @@ enum class BlobType : u32 {
class DataBase {
public:
static DataBase& Instance() {
return *Common::Singleton<DataBase>::Instance();
// spartanport: window close may race the first cache access during renderer startup.
static auto* const database = Common::Singleton<DataBase>::Instance();
return *database;
}
void Open();
void Close();
// spartanport: wait for queued and active writes; false if the timeout expires first.
bool Flush(std::chrono::milliseconds timeout);
[[nodiscard]] bool IsOpened() const {
return opened;
}
@@ -55,7 +62,8 @@ public:
private:
std::jthread io_worker{};
std::filesystem::path cache_path{};
bool opened{};
// spartanport: publish initialization safely to an exit hook on another thread.
std::atomic<bool> opened{};
};
} // namespace Storage
@@ -108,8 +108,14 @@ ComputePipeline::ComputePipeline(const Instance& instance, Scheduler& scheduler,
};
auto [pipeline_result, pipe] =
instance.GetDevice().createComputePipelineUnique(pipeline_cache, compute_pipeline_ci);
ASSERT_MSG(pipeline_result == vk::Result::eSuccess, "Failed to create compute pipeline: {}",
vk::to_string(pipeline_result));
// spartanport: a refused pipeline skips its dispatches instead of ending the game.
if (pipeline_result != vk::Result::eSuccess) {
LOG_ERROR(Render_Vulkan,
"Failed to create compute pipeline {:#x} ({}): {}; its dispatches are skipped",
std::hash<ComputePipelineKey>{}(compute_key), debug_str,
vk::to_string(pipeline_result));
return;
}
pipeline = std::move(pipe);
SetObjectName(device, *pipeline, "Compute Pipeline {}", debug_str);
RecordWaveSizes();
@@ -436,8 +436,13 @@ GraphicsPipeline::GraphicsPipeline(
auto [pipeline_result, pipe] =
CreateCachedGraphicsPipeline(device, pipeline_cache, pipeline_info, key,
instance.GetPipelineBinaryGlobalKey(), binary_directory);
ASSERT_MSG(pipeline_result == vk::Result::eSuccess, "Failed to create graphics pipeline: {}",
vk::to_string(pipeline_result));
// spartanport: a refused pipeline skips its draws instead of ending the game.
if (pipeline_result != vk::Result::eSuccess) {
LOG_ERROR(Render_Vulkan,
"Failed to create graphics pipeline {:#x} ({}): {}; its draws are skipped",
std::hash<GraphicsPipelineKey>{}(key), debug_str, vk::to_string(pipeline_result));
return;
}
pipeline = std::move(pipe);
SetObjectName(device, *pipeline, "Graphics Pipeline {}", debug_str);
RecordWaveSizes();
@@ -451,8 +456,15 @@ GraphicsPipeline::GraphicsPipeline(
auto [square_result, square] =
CreateCachedGraphicsPipeline(device, pipeline_cache, square_info, key,
instance.GetPipelineBinaryGlobalKey(), binary_directory);
ASSERT_MSG(square_result == vk::Result::eSuccess,
"Failed to create the squaring pipeline: {}", vk::to_string(square_result));
// spartanport: the main draw needs this pass too, so a refusal invalidates both.
if (square_result != vk::Result::eSuccess) {
LOG_ERROR(Render_Vulkan,
"Failed to create squaring pipeline {:#x} ({}): {}; its draws are skipped",
std::hash<GraphicsPipelineKey>{}(key), debug_str,
vk::to_string(square_result));
pipeline.reset();
return;
}
square_pipeline = std::move(square);
SetObjectName(device, *square_pipeline, "Graphics Pipeline {} squaring pass", debug_str);
}
@@ -6,6 +6,8 @@
#include <cstdlib>
#include <cstdio>
#include <cstring>
// spartanport: declare the exception type caught during speculative specialization.
#include <exception>
#include <mutex>
#include <ranges>
#include <string>
@@ -627,6 +629,12 @@ void PipelineCache::PublishGraphics(bool wait) {
continue;
}
const auto key = it.key();
// spartanport: keep a null refusal marker; do not publish stages or save a failed key.
if (!build->pipeline || !build->pipeline->IsValid()) {
graphics_pipelines.try_emplace(key);
it = pending_graphics.erase(it);
continue;
}
build->pipeline->PublishStages(build->live_infos);
const auto hash = std::hash<GraphicsPipelineKey>{}(key);
if (NewPipelineKey(fmt::format("g_{:#018x}", hash))) {
@@ -718,7 +726,9 @@ const GraphicsPipeline* PipelineCache::TryPreparedPipeline(const PreparedDraw& p
}
}
const auto it = graphics_pipelines.find(prepared.key);
return it != graphics_pipelines.end() ? it->second.get() : nullptr;
// spartanport: prepared draws must skip refused pipelines on every lookup.
return it != graphics_pipelines.end() && it->second && it->second->IsValid() ? it->second.get()
: nullptr;
}
const GraphicsPipeline* PipelineCache::GetGraphicsPipeline(const DrawIndirectParams params,
@@ -746,7 +756,8 @@ const GraphicsPipeline* PipelineCache::GetGraphicsPipeline(const DrawIndirectPar
if (AsyncCompilation() && !sel.regs->stage_enable.hs_en && !EmulatorSettings.IsShaderCollect()) {
const auto found = graphics_pipelines.find(sel.graphics_key);
if (found != graphics_pipelines.end()) {
return found->second.get();
// spartanport: a refused async build stays empty so later draws also skip it.
return found->second && found->second->IsValid() ? found->second.get() : nullptr;
}
if (!pending_graphics.contains(sel.graphics_key) && compiler->HasCapacity()) {
PhaseTimer capture{g_spartanport_snapshot_ns};
@@ -792,6 +803,13 @@ const GraphicsPipeline* PipelineCache::GetGraphicsPipeline(const DrawIndirectPar
instance, scheduler, desc_heap, profile, sel.graphics_key, *pipeline_cache, sel.infos,
sel.runtime_infos, sel.fetch_shader, sel.modules, sdata, false);
// spartanport: remember the refusal without retrying every draw or persisting its key.
if (!it.value()->IsValid()) {
it.value().reset();
sel.fetch_shader.reset();
return nullptr;
}
if (NewPipelineKey(fmt::format("g_{:#018x}", pipeline_hash))) {
RegisterPipelineData(sel.graphics_key, pipeline_hash, sdata);
}
@@ -807,7 +825,8 @@ const GraphicsPipeline* PipelineCache::GetGraphicsPipeline(const DrawIndirectPar
}
sel.fetch_shader.reset();
}
return it->second.get();
// spartanport: skip cached refusals on every synchronous lookup too.
return it->second && it->second->IsValid() ? it->second.get() : nullptr;
}
const ComputePipeline* PipelineCache::GetComputePipeline() {
@@ -828,6 +847,11 @@ const ComputePipeline* PipelineCache::GetComputePipeline() {
it.value() = std::make_unique<ComputePipeline>(instance, scheduler, desc_heap, profile,
*pipeline_cache, compute_key, *sel.infos[0],
sel.modules[0], sdata, false);
// spartanport: a null marker skips repeated dispatches and never saves a refused key.
if (!it.value()->IsValid()) {
it.value().reset();
return nullptr;
}
if (NewPipelineKey(fmt::format("c_{:#018x}", compute_key.value))) {
RegisterPipelineData(compute_key, sdata);
}
@@ -838,7 +862,8 @@ const ComputePipeline* PipelineCache::GetComputePipeline() {
module_related_pipelines[m].emplace_back(compute_key);
}
}
return it->second.get();
// spartanport: a cached empty handle is not a dispatchable pipeline.
return it->second && it->second->IsValid() ? it->second.get() : nullptr;
}
bool PipelineCache::RefreshGraphicsKey(PipelineSelection& sel) {
@@ -1136,6 +1161,19 @@ bool PipelineCache::RefreshGraphicsStages(PipelineSelection& sel) {
}
const auto* vs_info = sel.infos[static_cast<u32>(SwStage::Vertex)];
// spartanport: a draw with no vertex table has no input to build; skip before format lookup.
if (vs_info && sel.fetch_shader) {
for (const auto& attrib : sel.fetch_shader->attributes) {
if (vs_info->UdPointerNull(attrib.sgpr_base)) {
static std::atomic<u32> reported{0};
if (!sel.worker && reported.fetch_add(1, std::memory_order_relaxed) < 5) {
LOG_WARNING(Render_Vulkan,
"Draw skipped: its vertex buffer table pointer is null or short");
}
return false;
}
}
}
if (vs_info && sel.fetch_shader && !instance.IsVertexInputDynamicState()) {
// Without vertex input dynamic state, the pipeline needs to specialize on format.
// Stride will still be handled outside the pipeline using dynamic state.
@@ -1225,10 +1263,24 @@ PipelineCache::Result PipelineCache::GetProgram(PipelineSelection& sel, HwStage
return {};
}
runtime_fault_recover = &recover;
info.RefreshFlatBuf();
auto spec = Shader::StageSpecialization(info, runtime_info, profile, binding);
// spartanport: torn guest data can throw; abandon preparation like a fault, without
// changing the non-unwinding fault jump out of this try block.
std::optional<Shader::StageSpecialization> spec;
try {
info.RefreshFlatBuf();
spec.emplace(info, runtime_info, profile, binding);
} catch (const std::exception& e) {
runtime_fault_recover = nullptr;
static std::atomic<u32> logged{0};
if (logged.fetch_add(1, std::memory_order_relaxed) < 8) {
LOG_WARNING(Render_Vulkan, "Draw preparation ahead of the GPU thread stopped: {}",
e.what());
}
worker.failed = true;
return {};
}
runtime_fault_recover = nullptr;
const auto it = std::ranges::find(program->modules, spec, &Program::Module::spec);
const auto it = std::ranges::find(program->modules, *spec, &Program::Module::spec);
if (it == program->modules.end()) {
worker.failed = true;
return {};
@@ -36,6 +36,11 @@ public:
return *pipeline;
}
// spartanport: an empty handle marks a refused pipeline so callers can skip its work.
bool IsValid() const noexcept {
return static_cast<bool>(pipeline);
}
vk::PipelineLayout GetLayout() const noexcept {
return *pipeline_layout;
}
@@ -853,6 +853,10 @@ bool PipelineCache::LoadComputePipeline(Serialization::Archive& ar) {
auto pipeline = std::make_unique<ComputePipeline>(instance, scheduler, desc_heap,
profile, *pipeline_cache, key, *info,
module, sdata, true);
// spartanport: leave a null reservation so the failed-slot sweep can remove it.
if (!pipeline->IsValid()) {
return false;
}
std::scoped_lock lock{warmup_results_mutex};
warmup_compute.emplace_back(key, std::move(pipeline));
return true;
@@ -866,6 +870,11 @@ bool PipelineCache::LoadComputePipeline(Serialization::Archive& ar) {
sel.infos.fill(nullptr);
sel.modules.fill(nullptr);
// spartanport: drop refused preload entries so runtime can retry with the live shaders.
if (!warmup_pool && !it.value()->IsValid()) {
compute_pipelines.erase(compute_key);
return false;
}
return true;
}
@@ -941,6 +950,10 @@ bool PipelineCache::LoadGraphicsPipeline(Serialization::Archive& ar) {
auto pipeline = std::make_unique<GraphicsPipeline>(
instance, scheduler, desc_heap, profile, key, *pipeline_cache, infos,
runtime_infos, fetch, modules, sdata, true);
// spartanport: an invalid object would survive the null-slot sweep and be reused.
if (!pipeline->IsValid()) {
return false;
}
std::scoped_lock lock{warmup_results_mutex};
warmup_graphics.emplace_back(key, std::move(pipeline));
return true;
@@ -955,6 +968,11 @@ bool PipelineCache::LoadGraphicsPipeline(Serialization::Archive& ar) {
sel.modules.fill(nullptr);
sel.fetch_shader.reset();
// spartanport: a refused preload must not become a usable cache entry.
if (!warmup_pool && !it.value()->IsValid()) {
graphics_pipelines.erase(sel.graphics_key);
return false;
}
return true;
}
@@ -1086,24 +1104,31 @@ void PipelineCache::WarmUp(const std::function<void(u32, u32)>& progress) {
Serialization::Archive ar{std::move(data)};
Serialization::Reader pldata{ar};
u32 version{};
pldata.Read(version);
if (version != Serialization::PipelineKeyVersion) {
return;
}
// spartanport: skip damaged keys/metas and clear partial stage selections for retry.
try {
u32 version{};
pldata.Read(version);
if (version != Serialization::PipelineKeyVersion) {
return;
}
u32 is_compute{};
pldata.Read(is_compute);
u32 is_compute{};
pldata.Read(is_compute);
bool result{};
if (is_compute) {
result = LoadComputePipeline(ar);
} else {
result = LoadGraphicsPipeline(ar);
}
bool result{};
if (is_compute) {
result = LoadComputePipeline(ar);
} else {
result = LoadGraphicsPipeline(ar);
}
if (result) {
++num_pipelines;
if (result) {
++num_pipelines;
}
} catch (const std::exception&) {
sel.infos.fill(nullptr);
sel.modules.fill(nullptr);
sel.fetch_shader.reset();
}
});
@@ -1117,17 +1142,18 @@ void PipelineCache::WarmUp(const std::function<void(u32, u32)>& progress) {
for (auto& [key, pipeline] : warmup_compute) {
compute_pipelines[key] = std::move(pipeline);
}
for (auto it = graphics_pipelines.begin(); it != graphics_pipelines.end();) {
it = it->second ? std::next(it) : graphics_pipelines.erase(it);
}
for (auto it = compute_pipelines.begin(); it != compute_pipelines.end();) {
it = it->second ? std::next(it) : compute_pipelines.erase(it);
}
num_pipelines -= pool->failed;
warmup_graphics.clear();
warmup_compute.clear();
pool.reset();
}
// spartanport: discard failed serial reservations too so runtime can rebuild their pipelines.
for (auto it = graphics_pipelines.begin(); it != graphics_pipelines.end();) {
it = it->second ? std::next(it) : graphics_pipelines.erase(it);
}
for (auto it = compute_pipelines.begin(); it != compute_pipelines.end();) {
it = it->second ? std::next(it) : compute_pipelines.erase(it);
}
std::printf("Pipeline warm-up: %u of %u pipelines in %.2f s (%u threads)\n", num_pipelines,
num_total_pipelines,
std::chrono::duration<double>(std::chrono::steady_clock::now() - started).count(),
@@ -1351,13 +1377,18 @@ bool PipelineCache::LoadDiskProgram(u64 hash) {
bool have_info = false;
size_t inserted = 0;
for (auto& blob : *blobs) {
Serialization::Archive meta_ar{std::move(blob.meta)};
Shader::Info info{};
Shader::StageSpecialization spec{};
std::optional<Shader::Gcn::FetchShaderData> fetch;
size_t perm_idx{};
if (!LoadShaderMeta(meta_ar, info, fetch, spec, perm_idx) || perm_idx != blob.idx ||
info.pgm_hash != hash) {
// spartanport: a damaged permutation must not prevent loading the next saved shader.
try {
Serialization::Archive meta_ar{std::move(blob.meta)};
if (!LoadShaderMeta(meta_ar, info, fetch, spec, perm_idx) || perm_idx != blob.idx ||
info.pgm_hash != hash) {
continue;
}
} catch (const std::exception&) {
continue;
}
const auto module = CompileSPV(blob.spv, instance.GetDevice());
@@ -1520,7 +1551,14 @@ bool PipelineCache::StartBackgroundWarmUp() {
warm.total = u32(db.CountBlobs(Storage::BlobType::PipelineKey));
db.ForEachBlob(
Storage::BlobType::PipelineKey,
[&](std::vector<u8>&& data) { ReadWarmEntry(warm, std::move(data)); },
[&](std::vector<u8>&& data) {
// spartanport: skip damaged records individually so the reader job keeps going.
try {
ReadWarmEntry(warm, std::move(data));
} catch (const std::exception&) {
warm.done.fetch_add(1, std::memory_order_relaxed);
}
},
[&] { return warm.stopping.load(std::memory_order_relaxed); });
warm.stages.clear();
warm.reader_ns = ElapsedNs(reader_started);
@@ -1566,7 +1604,8 @@ void PipelineCache::ReadWarmEntry(BackgroundWarmup& warm, std::vector<u8>&& data
if (!hashes[i]) {
continue;
}
auto& stage = warm.stages[hashes[i]];
// spartanport: keep failed parses out of the shared stage cache for later records.
auto stage = warm.stages[hashes[i]];
if (!stage) {
stage = std::make_shared<WarmStage>();
stage->device = instance.GetDevice();
@@ -1598,6 +1637,7 @@ void PipelineCache::ReadWarmEntry(BackgroundWarmup& warm, std::vector<u8>&& data
if (!stage->module) {
stage->merged = true; // unusable: info stays null
}
warm.stages[hashes[i]] = stage;
}
entry.stages[i] = stage;
}
@@ -1726,10 +1766,19 @@ void PipelineCache::PumpWarmUp() {
built.compute = std::make_unique<ComputePipeline>(
instance, scheduler, desc_heap, profile, *pipeline_cache, compute_key,
*infos[0], modules[0], compute_data, true);
// spartanport: count refused background builds as failures and keep them
// unpublished.
if (!built.compute->IsValid()) {
return false;
}
} else {
built.graphics = std::make_unique<GraphicsPipeline>(
instance, scheduler, desc_heap, profile, graphics_key, *pipeline_cache, infos,
no_runtime_infos, fetch, modules, sdata, true);
// spartanport: leave a refused background graphics build out of the ready queue.
if (!built.graphics->IsValid()) {
return false;
}
}
warm.built.Push(std::move(built));
return true;
@@ -1858,6 +1907,11 @@ bool PersistentSrtInfo::Deserialize(Serialization::Archive& ar) {
srt.Read(this, sizeof(*this));
if (walker_func_size) {
// spartanport: reject truncated walker code before registering bytes beyond the record.
if (walker_func_size > ar.Remaining()) {
walker_func = nullptr;
throw Serialization::CorruptData{};
}
walker_func = RegisterWalkerCode(ar.CurrPtr(), walker_func_size);
ar.Advance(walker_func_size);
}
@@ -2,12 +2,13 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <boost/container/small_vector.hpp>
#include "common/alignment.h"
#include "spartanport_toggles.h"
#include "video_core/buffer_cache/buffer.h"
#include "video_core/renderer_vulkan/liverpool_to_vk.h"
#include "video_core/renderer_vulkan/vk_gpu_profiler.h"
#include "video_core/renderer_vulkan/vk_instance.h"
#include "video_core/renderer_vulkan/vk_runtime.h"
#include "video_core/renderer_vulkan/vk_gpu_profiler.h"
#include "video_core/renderer_vulkan/vk_scene_resolution.h"
#include "video_core/renderer_vulkan/vk_scheduler.h"
#include "video_core/texture_cache/blit_helper.h"
@@ -17,6 +18,16 @@
namespace Vulkan {
// spartanport: bound copy payloads to 64 KiB per command so large batches fit recording chunks.
template <typename T, typename Record>
static void RecordInSlices(Scheduler& scheduler, std::span<const T> regions, Record&& record) {
constexpr std::size_t PerCommand = 64 * 1024 / sizeof(T);
for (std::size_t at = 0; at < regions.size(); at += PerCommand) {
record(
scheduler.RecordData(regions.subspan(at, std::min(PerCommand, regions.size() - at))));
}
}
static vk::ImageType ConvertImageType(AmdGpu::ImageType type) noexcept {
switch (type) {
case AmdGpu::ImageType::Color1D:
@@ -142,9 +153,12 @@ void Runtime::CopyBuffer(const VideoCore::Buffer* src, const VideoCore::Buffer*
FlushBarriers();
}
scheduler.Record([src_handle = src->Handle(), dst_handle = dst->Handle(),
regions = scheduler.RecordData(copies)](vk::CommandBuffer cmdbuf) {
cmdbuf.copyBuffer(src_handle, dst_handle, regions.size(), regions.data());
// spartanport: split large buffer copy lists without changing their region order.
RecordInSlices(scheduler, copies, [&](std::span<const vk::BufferCopy> regions) {
scheduler.Record([src_handle = src->Handle(), dst_handle = dst->Handle(),
regions](vk::CommandBuffer cmdbuf) {
cmdbuf.copyBuffer(src_handle, dst_handle, regions.size(), regions.data());
});
});
if (bounded) {
@@ -243,10 +257,13 @@ void Runtime::UploadImage(VideoCore::Image* dst, const VideoCore::Buffer* src,
FlushBarriers();
}
scheduler.Record([src_handle = src->Handle(), image = dst->GetImage(),
regions = scheduler.RecordData(upload_copies)](vk::CommandBuffer cmdbuf) {
cmdbuf.copyBufferToImage(src_handle, image, vk::ImageLayout::eTransferDstOptimal,
regions.size(), regions.data());
// spartanport: keep image upload region payloads within the per-command recording budget.
RecordInSlices(scheduler, upload_copies, [&](std::span<const vk::BufferImageCopy> regions) {
scheduler.Record([src_handle = src->Handle(), image = dst->GetImage(),
regions](vk::CommandBuffer cmdbuf) {
cmdbuf.copyBufferToImage(src_handle, image, vk::ImageLayout::eTransferDstOptimal,
regions.size(), regions.data());
});
});
for (const auto& copy : upload_copies) {
@@ -302,10 +319,13 @@ void Runtime::DownloadImage(VideoCore::Image* src, const VideoCore::Buffer* dst,
FlushBarriers();
}
scheduler.Record([image = src->GetImage(), buffer = dst->Handle(),
regions = scheduler.RecordData(download_copies)](vk::CommandBuffer cmdbuf) {
cmdbuf.copyImageToBuffer(image, vk::ImageLayout::eTransferSrcOptimal, buffer,
regions.size(), regions.data());
// spartanport: image downloads need the same bounded recording payload as uploads.
RecordInSlices(scheduler, download_copies, [&](std::span<const vk::BufferImageCopy> regions) {
scheduler.Record(
[image = src->GetImage(), buffer = dst->Handle(), regions](vk::CommandBuffer cmdbuf) {
cmdbuf.copyImageToBuffer(image, vk::ImageLayout::eTransferSrcOptimal, buffer,
regions.size(), regions.data());
});
});
for (const auto& copy : download_copies) {
@@ -419,7 +439,6 @@ void Runtime::CopyImageWithBuffer(VideoCore::Image* src, VideoCore::Image* dst,
const VideoCore::Buffer* buffer, u64 offset) {
const u32 num_mips = std::min(src->info.resources.levels, dst->info.resources.levels);
const u32 num_layers = std::min(src->info.resources.layers, dst->info.resources.layers);
ASSERT(src->info.resources.layers == dst->info.resources.layers && num_mips == 1);
SetBackingSamples(dst, dst->info.num_samples, false);
SetBackingSamples(src, src->info.num_samples);
@@ -437,7 +456,22 @@ void Runtime::CopyImageWithBuffer(VideoCore::Image* src, VideoCore::Image* dst,
.imageOffset = {0, 0, 0},
.imageExtent = {src->info.size.width, src->info.size.height, src->info.size.depth},
};
const auto copy_size = BufferImageCopySize(buffer_copy, src->info.pixel_format);
// spartanport: copy the shared mips and layers, so a depth/colour copy without
// maintenance8 also handles images whose mip or layer counts differ.
u64 copy_size = BufferImageCopySize(buffer_copy, src->info.pixel_format);
boost::container::small_vector<vk::BufferImageCopy, 8> download_copies;
download_copies.push_back(buffer_copy);
for (u32 mip = 1; mip < num_mips; ++mip) {
auto mip_copy = buffer_copy;
mip_copy.bufferOffset = Common::AlignUp(offset + copy_size, 16);
mip_copy.imageSubresource.mipLevel = mip;
mip_copy.imageExtent = vk::Extent3D{std::max(src->info.size.width >> mip, 1u),
std::max(src->info.size.height >> mip, 1u),
std::max(src->info.size.depth >> mip, 1u)};
copy_size =
mip_copy.bufferOffset - offset + BufferImageCopySize(mip_copy, src->info.pixel_format);
download_copies.push_back(mip_copy);
}
scheduler.EndRendering();
@@ -451,13 +485,15 @@ void Runtime::CopyImageWithBuffer(VideoCore::Image* src, VideoCore::Image* dst,
FlushBarriers();
}
const auto download_copy = buffer_copy;
buffer_copy.imageSubresource.aspectMask = dst->aspect_mask & ~vk::ImageAspectFlagBits::eStencil;
auto upload_copies = download_copies;
for (auto& copy : upload_copies) {
copy.imageSubresource.aspectMask = dst->aspect_mask & ~vk::ImageAspectFlagBits::eStencil;
}
scheduler.Record([src_image = src->GetImage(), dst_image = dst->GetImage(),
handle = buffer->Handle(), download_copy,
upload_copy = buffer_copy](vk::CommandBuffer cmdbuf) {
handle = buffer->Handle(), download_copies = std::move(download_copies),
upload_copies = std::move(upload_copies)](vk::CommandBuffer cmdbuf) {
cmdbuf.copyImageToBuffer(src_image, vk::ImageLayout::eTransferSrcOptimal, handle,
download_copy);
download_copies);
const vk::MemoryBarrier2 post_copy_barrier = {
.srcStageMask = vk::PipelineStageFlagBits2::eCopy,
.srcAccessMask = vk::AccessFlagBits2::eTransferWrite,
@@ -470,7 +506,7 @@ void Runtime::CopyImageWithBuffer(VideoCore::Image* src, VideoCore::Image* dst,
.pMemoryBarriers = &post_copy_barrier,
});
cmdbuf.copyBufferToImage(handle, dst_image, vk::ImageLayout::eTransferDstOptimal,
upload_copy);
upload_copies);
});
dst->flags |= (src->flags & VideoCore::ImageFlagBits::GpuModified);
@@ -276,14 +276,17 @@ void Scheduler::KickRecording(bool force) {
if (full_chunks.empty() && record_chunk->Empty()) {
return;
}
// spartanport: acquire before handoff so recording storage stays valid; AcquireChunk
// takes recorder_mutex itself, so it must run before the lock below.
std::unique_ptr<RecordChunk> next = record_chunk->Empty() ? nullptr : AcquireChunk();
bool wake;
{
std::scoped_lock lk{recorder_mutex};
for (auto& chunk : full_chunks) {
recorder_queue.push_back(std::move(chunk));
}
if (!record_chunk->Empty()) {
recorder_queue.push_back(std::move(record_chunk));
if (next) {
recorder_queue.push_back(std::exchange(record_chunk, std::move(next)));
}
wake = recorder_sleeping;
queued_chunks.store(recorder_queue.size(), std::memory_order_release);
@@ -582,6 +582,32 @@ using SubmitFunc = Common::UniqueFunction<void, SubmitInfo&>;
class RecordChunk {
public:
static constexpr size_t Capacity = 128 * 1024;
// spartanport: keep the consuming command in the chunk that owns its spilled data.
static constexpr size_t CommandReserve = 1024;
// spartanport: oversized groups use chunk-owned storage instead of stopping with exit 23.
bool ReserveData(size_t bytes) {
const size_t remaining = Capacity - used;
if (remaining < CommandReserve) {
return false;
}
if (spill_remaining >= bytes) {
return true;
}
if (bytes <= remaining - CommandReserve) {
spill_cursor = nullptr;
spill_remaining = 0;
return true;
}
if (bytes <= Capacity - CommandReserve) {
return false;
}
std::unique_ptr<std::byte[]> block{new std::byte[bytes]};
spill_cursor = block.get();
spill_remaining = bytes;
spill_blocks.push_back(std::move(block));
return true;
}
/// Returns false (and leaves `func` untouched) when the chunk has no room.
template <typename Func>
@@ -606,8 +632,21 @@ public:
/// Raw storage in the chunk for a command's variable-length data; null when full.
void* Allocate(size_t bytes, size_t align) {
// spartanport: preserve alignment within the oversized group's spill block.
if (spill_cursor) {
void* result = spill_cursor;
size_t remaining = spill_remaining;
if (!std::align(align, bytes, result, remaining)) {
return nullptr;
}
spill_used += spill_remaining - remaining + bytes;
spill_cursor = static_cast<std::byte*>(result) + bytes;
spill_remaining = remaining - bytes;
return result;
}
const size_t offset = (used + align - 1) & ~(align - 1);
if (offset + bytes > Capacity) {
// spartanport: reject oversized inline allocations without overflowing the size check.
if (offset > Capacity || bytes > Capacity - offset) {
return nullptr;
}
used = offset + bytes;
@@ -624,6 +663,10 @@ public:
}
first = last = nullptr;
used = 0;
// spartanport: release spilled data only after every consuming command has executed.
spill_cursor = nullptr;
spill_remaining = spill_used = 0;
spill_blocks.clear();
}
[[nodiscard]] bool Empty() const noexcept {
@@ -631,7 +674,8 @@ public:
}
[[nodiscard]] size_t Size() const noexcept {
return used;
// spartanport: count spilled payloads toward recording batch thresholds.
return used + spill_used;
}
private:
@@ -662,6 +706,10 @@ private:
alignas(64) std::byte storage[Capacity];
size_t used = 0;
// spartanport: retain oversized groups until this chunk is recycled.
std::vector<std::unique_ptr<std::byte[]>> spill_blocks;
std::byte* spill_cursor = nullptr;
size_t spill_remaining = 0, spill_used = 0;
CommandBase* first{};
CommandBase* last{};
};
@@ -753,13 +801,20 @@ public:
return;
}
if (!record_chunk->Push(std::forward<Func>(func))) {
full_chunks.push_back(std::move(record_chunk));
record_chunk = AcquireChunk();
// spartanport: replace before retiring so recording storage stays valid.
RetireChunk();
const bool pushed = record_chunk->Push(std::forward<Func>(func));
ASSERT(pushed);
}
}
// spartanport: acquire the replacement first so reentrant recording during acquisition
// still finds a valid chunk, including guest memory faults handled on this thread.
void RetireChunk() {
auto next = AcquireChunk();
full_chunks.push_back(std::exchange(record_chunk, std::move(next)));
}
/// True when Record() defers commands (and RecordData() copies into chunks).
[[nodiscard]] bool IsRecordingDeferred() const noexcept {
return recording_threaded && !direct_mode &&
@@ -772,10 +827,12 @@ public:
if (!IsRecordingDeferred()) {
return;
}
ASSERT(bytes + 1024 <= RecordChunk::Capacity);
if (RecordChunk::Capacity - record_chunk->Size() < bytes + 1024) {
full_chunks.push_back(std::move(record_chunk));
record_chunk = AcquireChunk();
// spartanport: spill oversized groups while keeping their consumer in the same chunk.
if (!record_chunk->ReserveData(bytes)) {
// spartanport: replace before retiring so recording storage stays valid.
RetireChunk();
const bool reserved = record_chunk->ReserveData(bytes);
ASSERT_MSG(reserved, "Cannot reserve {} bytes in an empty recording chunk", bytes);
}
}
@@ -58,6 +58,10 @@ private:
/// game compiled the same one (or the reverse) is dropped. True when `value` was stored.
template <typename Map, typename Key, typename Value>
bool PublishFirst(Map& map, const Key& key, Value&& value) {
// spartanport: only usable pipelines can win publication; keep null runtime refusal markers.
if (!value || !value->IsValid()) {
return false;
}
return map.try_emplace(key, std::forward<Value>(value)).second;
}
@@ -62,12 +62,29 @@ TextureCache::TextureCache(const Vulkan::Instance& instance_, Vulkan::Scheduler&
const s64 mem_threshold = std::min<s64>(device_local_memory, TARGET_GC_THRESHOLD);
const s64 min_vacancy_expected = (6 * mem_threshold) / 10;
const s64 min_vacancy_critical = (2 * mem_threshold) / 10;
// spartanport: scale GC floors to avoid oversized defaults on small device budgets.
const s64 min_pressure_floor =
std::clamp<s64>(device_local_memory / 4, 256_MB, DEFAULT_PRESSURE_GC_MEMORY);
const s64 min_critical_floor =
std::clamp<s64>(device_local_memory / 2, 512_MB, DEFAULT_CRITICAL_GC_MEMORY);
pressure_gc_memory = static_cast<u64>(
std::max<u64>(std::min(device_local_memory - min_vacancy_expected, min_spacing_expected),
DEFAULT_PRESSURE_GC_MEMORY));
std::max<s64>(std::min(device_local_memory - min_vacancy_expected, min_spacing_expected),
min_pressure_floor));
critical_gc_memory = static_cast<u64>(
std::max<u64>(std::min(device_local_memory - min_vacancy_critical, min_spacing_critical),
DEFAULT_CRITICAL_GC_MEMORY));
std::max<s64>(std::min(device_local_memory - min_vacancy_critical, min_spacing_critical),
min_critical_floor));
// spartanport: below an 8 GiB budget, leave textures more room before pressure GC, capped at
// critical. SPARTAN_GC_PRESSURE_PERCENT defaults to 60 (clamped to 10-75).
if (device_local_memory < TARGET_GC_THRESHOLD) {
static const s64 percent = [] {
const char* env = std::getenv("SPARTAN_GC_PRESSURE_PERCENT");
return std::clamp<s64>(env ? std::strtoll(env, nullptr, 10) : 60, 10, 75);
}();
pressure_gc_memory =
std::min(std::max<u64>(pressure_gc_memory,
static_cast<u64>(device_local_memory / 100 * percent)),
critical_gc_memory);
}
trigger_gc_memory = static_cast<u64>((device_local_memory - mem_threshold) / 2);
}
@@ -712,7 +729,8 @@ ImageId TextureCache::ResolveDepthOverlap(const ImageInfo& requested_info, Bindi
const bool bpp_match = requested_info.num_bits == cache_image.info.num_bits;
// If an image in the cache has less slices we need to expand it
bool recreate = cache_image.info.resources < requested_info.resources;
// spartanport: recreate if either dimension is too small for the requested subresources.
bool recreate = !cache_image.info.resources.Contains(requested_info.resources);
switch (binding) {
case BindingType::Texture:
@@ -746,18 +764,26 @@ ImageId TextureCache::ResolveDepthOverlap(const ImageInfo& requested_info, Bindi
if (recreate) {
auto new_info = requested_info;
new_info.resources = std::max(requested_info.resources, cache_image.info.resources);
// spartanport: take the larger count per dimension; the ordered max could keep the cached
// image's extra mips and still drop a requested layer.
new_info.resources = {
.levels = std::max(requested_info.resources.levels, cache_image.info.resources.levels),
.layers = std::max(requested_info.resources.layers, cache_image.info.resources.layers),
};
new_info.UpdateSize();
const auto new_image_id = slot_images.insert(instance, runtime, slot_image_views, new_info);
RegisterImage(new_image_id);
// spartanport: reacquire the old image after insert before inheriting or copying it.
auto& old_image = slot_images[cache_image_id];
// Inherit image usage
auto& new_image = slot_images[new_image_id];
new_image.usage = cache_image.usage;
new_image.usage = old_image.usage;
new_image.flags &= ~ImageFlagBits::Dirty;
// When creating a depth buffer through overlap resolution don't clear it on first use.
new_image.info.meta_info.htile_clear_mask = 0;
runtime.CopyColorAndDepth(&cache_image, &new_image);
runtime.CopyColorAndDepth(&old_image, &new_image);
// Free the cache image.
FreeImage(cache_image_id);
@@ -847,17 +873,19 @@ std::tuple<ImageId, int, int> TextureCache::ResolveOverlap(const ImageInfo& imag
return {ExpandImage(image_info, cache_image_id), -1, -1};
}
const bool pow2_padding_only =
image_info.props.is_pow2 != cache_image.info.props.is_pow2 &&
// spartanport: reuse identical single-level surfaces when only tiler padding differs.
const bool padding_only =
image_info.tile_mode == cache_image.info.tile_mode &&
image_info.size == cache_image.info.size &&
image_info.pitch == cache_image.info.pitch && image_info.resources.levels == 1 &&
cache_image.info.resources.levels == 1 && image_info.resources.layers == 1 &&
cache_image.info.resources.layers == 1;
cache_image.info.resources.layers == 1 &&
(image_info.props.is_pow2 != cache_image.info.props.is_pow2 ||
image_info.type == cache_image.info.type);
// Size and resources are less than or equal, use image view.
if (image_info.pixel_format != cache_image.info.pixel_format ||
image_info.guest_size <= cache_image.info.guest_size || pow2_padding_only) {
image_info.guest_size <= cache_image.info.guest_size || padding_only) {
auto result_id = merged_image_id ? merged_image_id : cache_image_id;
const auto& result_image = slot_images[result_id];
const bool is_compatible =
@@ -866,8 +894,9 @@ std::tuple<ImageId, int, int> TextureCache::ResolveOverlap(const ImageInfo& imag
}
// Size and resources are greater, expand the image.
// spartanport: expand if either dimension is missing, regardless of mip ordering.
if (image_info.type == cache_image.info.type &&
image_info.resources > cache_image.info.resources) {
!cache_image.info.resources.Contains(image_info.resources)) {
return {ExpandImage(image_info, cache_image_id), -1, -1};
}
@@ -1060,6 +1089,26 @@ ImageId TextureCache::ExpandImage(const ImageInfo& info, ImageId image_id) {
return new_image_id;
}
// spartanport: clamp views to the resolved image to prevent invalid views and barrier asserts.
static void ClampViewRange(TextureCache::ImageDesc& desc, const Image& image) {
auto& range = desc.view_info.range;
const auto requested = range;
if (!range.ClampTo(image.info.resources)) {
return;
}
static std::atomic<u32> reported{0};
if (reported.fetch_add(1, std::memory_order_relaxed) < 8) {
std::printf("Texture cache: view mips %u+%u layers %u+%u clamped to image %#llx "
"(%u mips, %u layers, %s; requested %s)\n",
requested.base.level, requested.extent.levels, requested.base.layer,
requested.extent.layers, (unsigned long long)image.info.guest_address,
std::max(image.info.resources.levels, 1u),
std::max(image.info.resources.layers, 1u),
vk::to_string(image.info.pixel_format).c_str(),
vk::to_string(desc.info.pixel_format).c_str());
}
}
ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_fmt) {
const auto& info = desc.info;
ASSERT(info.guest_address != 0);
@@ -1101,6 +1150,8 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_fmt) {
if (cached.view_slice > 0) {
desc.view_info.range.base.layer = cached.view_slice;
}
// spartanport: cached images must also bound descriptor mip and layer ranges.
ClampViewRange(desc, image);
return cached.image_id;
}
@@ -1178,8 +1229,8 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_fmt) {
// Cannot reuse this image as we need the exact requested format.
image_id = {};
created.reason = "exact format";
} else if (image_resolved.info.resources < info.resources) {
// The image was clearly picked up wrong.
} else if (!image_resolved.info.resources.Contains(info.resources)) {
// spartanport: reject images missing mips or layers before creating an invalid view.
FreeImage(image_id);
image_id = {};
LOG_WARNING(Render_Vulkan, "Image overlap resolve failed");
@@ -1220,6 +1271,9 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_fmt) {
desc.view_info.range.base.layer = view_slice;
}
// spartanport: overlap offsets can leave the view past the resolved image.
ClampViewRange(desc, image);
cached = FindImageCacheEntry{
.address = info.guest_address,
.size = info.guest_size,
@@ -1362,10 +1416,12 @@ ImageView& TextureCache::FindDepthTarget(ImageId image_id, const ImageDesc& desc
}
Image& stencil_image = slot_images[stencil_id];
TouchImage(stencil_image);
stencil_image.AssociateDepth(image_id, image.image_uid);
// spartanport: reacquire the depth image after a possible stencil insertion.
stencil_image.AssociateDepth(image_id, slot_images[image_id].image_uid);
}
return image.FindView(desc.view_info, false);
// spartanport: return a view through the current slot after a possible stencil insertion.
return slot_images[image_id].FindView(desc.view_info, false);
}
void TextureCache::RefreshImage(Image& image) {
@@ -1687,6 +1743,8 @@ void TextureCache::GarbageCollectImages() {
bool aggresive = false;
u64 ticks_to_destroy = 0;
size_t num_deletions = 0;
// spartanport: bound work even when most old images cannot be evicted.
u32 visited = 0;
const auto configure = [&](bool allow_aggressive) {
pressured = total_used_memory >= pressure_gc_memory;
@@ -1694,24 +1752,43 @@ void TextureCache::GarbageCollectImages() {
ticks_to_destroy = aggresive ? 160 : pressured ? 80 : 16;
ticks_to_destroy = std::min(ticks_to_destroy, gc_tick);
num_deletions = aggresive ? 40 : pressured ? 20 : 10;
// spartanport: each pass has its own visit limit and kept-image report.
visited = 0;
gc_kept = gc_kept_bytes = 0;
};
const auto clean_up = [&](ImageId image_id) {
if (num_deletions == 0) {
// spartanport: stop at the deletion budget or the visit limit.
if (num_deletions == 0 || ++visited > 256) {
return true;
}
--num_deletions;
auto& image = slot_images[image_id];
ResolveReadbacks(image.info.guest_address, image.info.guest_size, true,
ReadbackSource::CacheCleanup);
const bool download = image.SafeToDownload();
const bool tiled = image.info.IsTiled();
if (tiled && download) {
// spartanport: a neighboring CPU write can leave the only valid contents on the GPU.
// GPU-dirty images are not kept; their data is in the buffer cache.
// Keep unresolved images below critical; test usage so the first pass can free them too.
const bool gpu_contents_unresolved =
True(image.flags & ImageFlagBits::GpuModified) && !download &&
False(image.flags & ImageFlagBits::GpuDirty) &&
False(image.flags & ImageFlagBits::CpuDirty) && total_used_memory < critical_gc_memory;
if ((tiled && download) || gpu_contents_unresolved) {
// This is a workaround for now. We can't handle non-linear image downloads.
// spartanport: report kept tiled or unresolved images and move them behind evictable
// entries.
++gc_kept;
gc_kept_bytes += image.info.guest_size;
lru_cache.Touch(image.lru_id, gc_tick);
return false;
}
if (download && !pressured) {
// spartanport: kept linear images must not block the next pass either.
lru_cache.Touch(image.lru_id, gc_tick);
return false;
}
// spartanport: only actual evictions spend the deletion budget.
--num_deletions;
if (download) {
// spartanport: synchronously, while the image still protects its pages. A deferred
// write-back landed after FreeImage had unprotected them, over whatever the game
@@ -1748,12 +1825,15 @@ void TextureCache::GarbageCollectImages() {
if (pressured || gc_downloads != 0) {
const auto now = std::chrono::steady_clock::now();
if (now - gc_report_time >= std::chrono::seconds(5)) {
// spartanport: expose tiled or unresolved images that the latest pass cannot evict.
std::printf("Texture cache: memory pressure, %llu of %llu MiB (critical %llu): "
"%llu images evicted, %llu written back since the last report\n",
"%llu images evicted, %llu written back since the last report; "
"%llu old GPU-written images kept (%llu MiB)\n",
(unsigned long long)(total_used_memory >> 20),
(unsigned long long)(pressure_gc_memory >> 20),
(unsigned long long)(critical_gc_memory >> 20),
(unsigned long long)gc_evictions, (unsigned long long)gc_downloads);
(unsigned long long)gc_evictions, (unsigned long long)gc_downloads,
(unsigned long long)gc_kept, (unsigned long long)(gc_kept_bytes >> 20));
gc_report_time = now;
gc_evictions = gc_downloads = 0;
}
@@ -1783,7 +1863,10 @@ void TextureCache::GarbageCollectSamplers() {
return true;
}
--num_deletions;
const size_t lru_id = samplers.at(hash).lru_id;
// spartanport: keep evicted samplers alive until the GPU completes their recorded work.
auto& sampler = samplers.at(hash);
const size_t lru_id = sampler.lru_id;
scheduler.DeferOperation([retired = std::move(sampler)] { (void)retired; });
samplers.erase(hash);
sampler_lru_cache.Free(lru_id);
return false;
@@ -456,6 +456,8 @@ private:
std::unordered_set<ImageId> download_images;
u64 total_used_memory = 0;
u64 gc_evictions = 0, gc_downloads = 0; ///< spartanport: pressure report
// spartanport: count old tiled images retained by the latest pass for pressure diagnostics.
u64 gc_kept = 0, gc_kept_bytes = 0;
std::chrono::steady_clock::time_point gc_report_time{};
u64 trigger_gc_memory = 0;
u64 pressure_gc_memory = 0;
@@ -3,6 +3,8 @@
#pragma once
// spartanport: clamp view ranges with standard min/max helpers.
#include <algorithm>
#include "common/slot_vector.h"
#include "common/types.h"
@@ -64,6 +66,11 @@ struct SubresourceExtent {
u32 layers = 1;
auto operator<=>(const SubresourceExtent&) const = default;
// spartanport: check both dimensions so extra mips cannot hide missing layers.
bool Contains(const SubresourceExtent& other) const noexcept {
return levels >= other.levels && layers >= other.layers;
}
};
struct SubresourceRange {
@@ -71,6 +78,21 @@ struct SubresourceRange {
SubresourceExtent extent;
auto operator<=>(const SubresourceRange&) const = default;
// spartanport: bound views to the resolved image, keeping counts positive without overflow.
bool ClampTo(const SubresourceExtent& resources) noexcept {
const u32 levels = std::max(resources.levels, 1u);
const u32 layers = std::max(resources.layers, 1u);
if (base.level < levels && base.layer < layers && extent.levels > 0 && extent.layers > 0 &&
extent.levels <= levels - base.level && extent.layers <= layers - base.layer) {
return false;
}
base.level = std::min(base.level, levels - 1);
base.layer = std::min(base.layer, layers - 1);
extent.levels = std::clamp(extent.levels, 1u, levels - base.level);
extent.layers = std::clamp(extent.layers, 1u, layers - base.layer);
return true;
}
};
struct ImageCopy {
+73
View File
@@ -0,0 +1,73 @@
// SPDX-License-Identifier: GPL-2.0-or-later
// spartanport: reserve storage for Common::SlotVector so entries stay fixed as the pool grows.
#include <cstddef>
#include <cstdio>
#include <cstdlib>
#include "common/slot_vector.h"
#ifdef _WIN32
#include <windows.h>
#else
#include <sys/mman.h>
#include <unistd.h>
#endif
namespace Common {
namespace {
[[noreturn]] void PageFailure(const char* operation) noexcept {
std::fprintf(stderr, "SlotVector: %s failed\n", operation);
std::abort();
}
} // namespace
void* SlotVectorReservePages(std::size_t size) noexcept {
#ifdef _WIN32
void* const base = VirtualAlloc(nullptr, size, MEM_RESERVE, PAGE_NOACCESS);
if (!base) {
PageFailure("address-space reservation");
}
#else
const long page_size = sysconf(_SC_PAGESIZE);
if (page_size <= 0 || size % static_cast<std::size_t>(page_size) != 0) {
PageFailure("reservation alignment");
}
void* const base = mmap(nullptr, size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (base == MAP_FAILED) {
PageFailure("address-space reservation");
}
#endif
return base;
}
void SlotVectorCommitPages(void* address, std::size_t size) noexcept {
#ifdef _WIN32
if (VirtualAlloc(address, size, MEM_COMMIT, PAGE_READWRITE) != address) {
PageFailure("page commit");
}
#else
// Keep the owned reservation; no fixed-address replacement mapping is needed.
if (mprotect(address, size, PROT_READ | PROT_WRITE) != 0) {
PageFailure("page commit");
}
#endif
}
void SlotVectorReleasePages(void* base, [[maybe_unused]] std::size_t size) noexcept {
if (!base) {
return;
}
#ifdef _WIN32
if (!VirtualFree(base, 0, MEM_RELEASE)) {
PageFailure("page release");
}
#else
if (munmap(base, size) != 0) {
PageFailure("page release");
}
#endif
}
} // namespace Common
+19
View File
@@ -14,6 +14,7 @@
#include <boost/asio/io_context.hpp>
#include "common/polyfill_thread.h"
#include "video_core/renderdoc.h"
#include "video_core/cache_storage.h"
#include <cstring>
#include <mutex>
#include <string>
@@ -211,6 +212,23 @@ static void StartProfileWriter() {
}
#endif
extern "C" void spartanportgpu_flush_cache(void) {
const char* env = std::getenv("SPARTAN_CACHE_FLUSH_ON_EXIT");
static const bool enabled = !(env && env[0] == '0');
auto& cache = Storage::DataBase::Instance();
if (!enabled || !cache.IsOpened()) {
return;
}
const auto start = std::chrono::steady_clock::now();
const bool done = cache.Flush(std::chrono::seconds(2));
const auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now() - start)
.count();
std::printf("Shader cache: %s at exit in %lld ms\n",
done ? "queue drained" : "queued writes NOT finished",
static_cast<long long>(ms));
}
extern "C" int spartanportgpu_init(const SpartanGpuConfig* config) {
SpartanSettings::Load();
#ifdef SPARTAN_PGO_GENERATE
@@ -240,6 +258,7 @@ extern "C" int spartanportgpu_init(const SpartanGpuConfig* config) {
}
LOG_INFO(Frontend, "Window closed by user");
runtime_report();
spartanportgpu_flush_cache();
std::fflush(stdout);
std::_Exit(0);
});
+3
View File
@@ -18,6 +18,9 @@ typedef struct {
void spartanportgpu_register_kernel(void);
/* Creates window, Vulkan device, presenter and GPU command processor. */
int spartanportgpu_init(const SpartanGpuConfig *config);
/* Unless SPARTAN_CACHE_FLUSH_ON_EXIT=0: waits at most two seconds for queued shader-cache
* writes before window close, guest exit or restart skips destructors. */
void spartanportgpu_flush_cache(void);
/* Function for an imported NID ("NID#lib#mod"), or 0 when the GPU library does not provide it. */
uintptr_t spartanportgpu_resolve(const char *scoped_nid);
/* Called first by the loader's SIGSEGV handler: 1 when a GPU page-tracking fault was handled. */
+2
View File
@@ -891,6 +891,7 @@ static void install_graphics(Segment *segments, uint64_t count, uint64_t size) {
* does: GPU, audio and guest threads are still running, so no destructors. */
static void menu_quit(void) {
puts("Pause menu: Exit to Desktop");
spartanportgpu_flush_cache();
fflush(NULL);
#ifdef _WIN32
TerminateProcess(GetCurrentProcess(),0);
@@ -967,6 +968,7 @@ static void apply_patches(const char *path, Segment *segments, uint64_t ns, cons
* applied at start). Descriptors are closed first so the old GPU device and its memory are
* released before the new process opens its own. */
void runtime_restart(void) {
spartanportgpu_flush_cache();
fflush(NULL);
#ifdef _WIN32
/* run.py sets SPARTAN_RESTART_COMMAND: the launch command line, started again (new patches). */
+171
View File
@@ -0,0 +1,171 @@
// SPDX-License-Identifier: GPL-2.0-or-later
// Run: out/gpu/cache-storage-test <dir> (a new directory that the test creates and removes).
// Exercise the production IO worker without a window, guest process or Vulkan device.
#include <atomic>
#include <barrier>
#include <cassert>
#include <chrono>
#include <cstdio>
#include <filesystem>
#include <fstream>
#include <future>
#include <thread>
#include <vector>
#include "../gpu/spartanportgpu.h"
// Include the implementation so optimized builds need not export its internal template writer.
#include "../gpu/shadps4/video_core/cache_storage.cpp"
// Mirror the production metadata shim without starting spartanportgpu_init's window/device.
namespace Core {
class Emulator {
public:
static void FillElfInfo(const SpartanGpuConfig& config) {
auto& info = Common::ElfInfo::Instance();
info.initialized = true;
info.game_serial = config.serial ? config.serial : "UNKNOWN";
info.title = config.title ? config.title : "";
info.sdk_ver = config.sdk_version;
info.psf_attributes.raw = config.psf_attributes;
}
};
} // namespace Core
int main(int argc, char** argv) {
assert(argc == 2); // Supply a new, test-owned directory inside the workspace.
const auto root = std::filesystem::absolute(argv[1]);
assert(!std::filesystem::exists(root));
std::filesystem::create_directories(root);
// Window close and renderer startup can make the first database access concurrently.
std::vector<Storage::DataBase*> databases(8);
std::barrier start{8};
std::vector<std::thread> readers;
for (unsigned i = 0; i < databases.size(); ++i) {
readers.emplace_back([&, i] {
start.arrive_and_wait();
databases[i] = &Storage::DataBase::Instance();
});
}
for (auto& reader : readers) {
reader.join();
}
for (auto* database : databases) {
assert(database == databases.front());
}
const auto save = [&](unsigned i) {
assert(Storage::WriteVector(Storage::BlobType::ShaderBinary, root / std::to_string(i),
std::vector<u32>{i, i + 1}));
};
const auto verify = [&](unsigned count) {
for (unsigned i = 0; i < count; ++i) {
const auto path = root / (std::to_string(i) + ".spv");
assert(std::filesystem::file_size(path) == 2 * sizeof(u32));
std::ifstream file(path, std::ios::binary);
u32 words[2]{};
file.read(reinterpret_cast<char*>(words), sizeof(words));
assert(file && words[0] == i && words[1] == i + 1);
assert(!std::filesystem::exists(path.string() + ".tmp"));
}
};
{
std::jthread worker{Storage::ProcessIO};
std::vector<std::thread> producers;
for (unsigned t = 0; t < 4; ++t) {
producers.emplace_back([&, t] {
for (unsigned i = 0; i < 100; ++i) {
save(t * 100 + i);
}
});
}
for (auto& producer : producers) {
producer.join();
}
} // Request stop immediately; every accepted write must finish before the join returns.
verify(400);
// Even a stop requested before the worker starts must drain its accepted backlog.
for (unsigned i = 400; i < 500; ++i) {
save(i);
}
std::stop_source stopped;
stopped.request_stop();
std::thread worker{[&] { Storage::ProcessIO(stopped.get_token()); }};
worker.join();
verify(500);
// Existing cache words still load; a miss must discard the caller's previous contents.
std::vector<u32> words{99};
auto path = root / "1";
Storage::LoadVector(Storage::BlobType::ShaderBinary, path, words);
assert((words == std::vector<u32>{1, 2}));
path = root / "absent";
Storage::LoadVector(Storage::BlobType::ShaderBinary, path, words);
assert(words.empty());
// Reject both an incomplete first word and a complete word followed by stray bytes.
for (const size_t bytes : {3, 5}) {
path = root / "bad.spv";
{
std::ofstream file(path, std::ios::binary);
file << std::string(bytes, 'x');
assert(file);
}
words = {99};
Storage::LoadVector(Storage::BlobType::ShaderBinary, path, words);
assert(words.empty());
std::vector<u8> raw{99};
Storage::LoadVector(Storage::BlobType::ShaderBinary, path, raw);
assert(raw == std::vector<u8>(bytes, 'x'));
}
path = root / "empty.spv";
std::ofstream{path, std::ios::binary}.close();
words = {99};
Storage::LoadVector(Storage::BlobType::ShaderBinary, path, words);
assert(words.empty());
// A cache root replaced by a file disables caching without throwing, then permits a retry.
Core::Emulator::FillElfInfo(SpartanGpuConfig{.serial = "TEST00000"});
using namespace Common::FS;
const auto original_cache = GetUserPath(PathType::CacheDir);
const auto cache_root = root / "cache";
std::filesystem::create_directory(cache_root);
SetUserPath(PathType::CacheDir, cache_root);
assert(std::filesystem::remove(cache_root));
std::ofstream{cache_root} << "blocked";
auto& cache = Storage::DataBase::Instance();
cache.Open();
assert(!cache.IsOpened());
assert(!cache.Save(Storage::BlobType::ShaderBinary, "blocked", std::vector<u32>{1}));
assert(std::filesystem::is_regular_file(cache_root));
assert(std::filesystem::remove(cache_root));
std::filesystem::create_directory(cache_root);
cache.Open();
assert(cache.IsOpened());
assert(std::filesystem::is_directory(cache_root / "TEST00000"));
// Flush waits while a write is queued or running and returns once the worker is idle.
std::promise<void> release;
std::atomic<bool> ran{false};
{
std::scoped_lock lock{m_request};
req_queue.emplace([&ran, gate = release.get_future().share()] {
gate.wait();
ran = true;
});
}
request_cv.notify_one();
assert(!cache.Flush(std::chrono::milliseconds(50)));
release.set_value();
assert(cache.Flush(std::chrono::seconds(5)) && ran);
// A saved blob is on disk when Flush returns; a closed cache has nothing to flush.
assert(cache.Save(Storage::BlobType::ShaderBinary, "flushed", std::vector<u32>{7}));
assert(cache.Flush(std::chrono::seconds(5)));
assert(std::filesystem::file_size(cache_root / "TEST00000" / "flushed.spv") == sizeof(u32));
cache.Close();
assert(cache.Flush(std::chrono::seconds(5)));
SetUserPath(PathType::CacheDir, original_cache);
std::filesystem::remove_all(root); // Only the directory this test creates above.
std::puts("Cache storage: concurrent writes, stop drain, flush and load hardening PASS");
}
+86
View File
@@ -0,0 +1,86 @@
// SPDX-License-Identifier: GPL-2.0-or-later
//
// Platform::IrqController is shared by guest threads (sceGnmAddEqEvent registers handlers) and
// the GPU thread (Signal on end-of-pipe, flip and idle). Each controller here starts empty, so
// every Register and Signal can be the first use of its interrupt and the controller's storage
// is filled from two threads at once. Before the fix that storage was an unsynchronised
// unordered_map: a concurrent insert could rehash it under the other thread and corrupt the
// heap, which in the game surfaced as crashes in unrelated structures soon after launch.
#include <atomic>
#include <cassert>
#include <cstdio>
#include <memory>
#include <thread>
#include <vector>
#include "core/platform.h"
using Platform::InterruptId;
static void TestHandlers() {
Platform::IrqController irq;
int uid{};
for (const auto id : {InterruptId::Compute0RelMem, InterruptId::GfxEop}) {
int persistent{};
int once{};
irq.Register(id, [&](InterruptId signaled) {
assert(signaled == id);
++persistent;
}, &uid);
irq.RegisterOnce(id, [&](InterruptId signaled) {
assert(signaled == id);
assert(once == 0);
++once;
});
irq.RegisterOnce(id, [&](InterruptId signaled) {
assert(signaled == id);
assert(once == 1);
++once;
});
irq.Signal(id);
assert(persistent == 1 && once == 1);
irq.Unregister(id, &uid);
irq.Signal(id);
irq.Signal(id);
assert(persistent == 1 && once == 2);
}
}
int main() {
TestHandlers();
constexpr InterruptId ids[] = {
InterruptId::Compute0RelMem, InterruptId::Compute1RelMem, InterruptId::Compute2RelMem,
InterruptId::Compute3RelMem, InterruptId::Compute4RelMem, InterruptId::Compute5RelMem,
InterruptId::Compute6RelMem, InterruptId::GfxFlip, InterruptId::GpuIdle,
InterruptId::GfxEop};
std::atomic<int> handled{0};
for (int round = 0; round < 2000; ++round) {
auto irq = std::make_unique<Platform::IrqController>();
std::atomic<bool> go{false};
std::vector<int> uids(std::size(ids));
std::thread guest([&] {
while (!go.load()) {
}
for (size_t i = 0; i < std::size(ids); ++i) {
irq->Register(ids[i], [&](InterruptId) { ++handled; }, &uids[i]);
}
});
std::thread gpu([&] {
while (!go.load()) {
}
for (size_t i = std::size(ids); i-- > 0;) {
irq->Signal(ids[i]);
}
});
go = true;
guest.join();
gpu.join();
// Every handler is registered exactly once and still fires after the race.
const int before = handled.load();
for (const auto id : ids) {
irq->Signal(id);
}
assert(handled.load() == before + int(std::size(ids)));
}
assert(handled.load() >= 2000 * int(std::size(ids)));
std::puts("PASS: IRQ controller survives concurrent first use");
}
+128
View File
@@ -0,0 +1,128 @@
// SPDX-License-Identifier: GPL-2.0-or-later
// Deferred payload ownership, grouped reservations and recycling, without a Vulkan device.
#include <algorithm>
#include <cassert>
#include <cstdio>
#include <cstring>
#include <limits>
#include <span>
#include <vector>
#include "video_core/renderer_vulkan/vk_scheduler.h"
extern "C" {
std::uint64_t runtime_disabled_optimizations = SpartanToggle::RecordPrefetch;
}
template <typename T>
static std::span<const T> Save(Vulkan::RecordChunk& chunk, std::span<const T> data) {
// Exercise the same reservation contract as Scheduler::RecordData, without a device.
assert(chunk.ReserveData(data.size_bytes() + alignof(T)));
auto* saved = static_cast<T*>(chunk.Allocate(data.size_bytes(), alignof(T)));
assert(saved);
std::memcpy(saved, data.data(), data.size_bytes());
return {saved, data.size()};
}
int main() {
using Vulkan::RecordChunk;
constexpr size_t Count = RecordChunk::Capacity / sizeof(vk::BufferCopy) + 257;
struct alignas(64) Marker {
unsigned value;
};
RecordChunk chunk;
unsigned executed = 0;
for (unsigned pass = 0; pass < 12; ++pass) {
std::vector<vk::BufferCopy> regions(Count);
for (size_t i = 0; i < Count; ++i) {
regions[i] = vk::BufferCopy{.srcOffset = i * 4, .dstOffset = i * 8, .size = pass + 1};
}
// Earlier commands must survive allocating and releasing spill blocks.
for (unsigned i = 0; i < 4000; ++i) {
assert(chunk.Push([](vk::CommandBuffer) {}));
}
if (pass % 2 == 0) {
const auto saved = Save(chunk, std::span<const vk::BufferCopy>{regions});
assert(chunk.Size() > RecordChunk::Capacity);
assert(chunk.Push([saved, pass, &executed](vk::CommandBuffer) {
for (size_t i = 0; i < saved.size(); ++i) {
assert(saved[i].srcOffset == i * 4);
assert(saved[i].dstOffset == i * 8);
assert(saved[i].size == pass + 1);
}
++executed;
}));
} else {
const auto half = Count / 2;
assert(chunk.ReserveData(regions.size() * sizeof(regions[0]) + sizeof(Marker) + 256));
const auto first = Save(chunk, std::span<const vk::BufferCopy>{regions.data(), half});
const Marker marker{.value = pass};
const auto aligned = Save(chunk, std::span<const Marker>{&marker, 1});
const auto second =
Save(chunk, std::span<const vk::BufferCopy>{regions.data() + half, Count - half});
assert(reinterpret_cast<uintptr_t>(aligned.data()) % alignof(Marker) == 0);
assert(chunk.Push([first, second, aligned, pass, &executed](vk::CommandBuffer) {
assert(aligned[0].value == pass);
for (size_t i = 0; i < Count; ++i) {
const auto& region = i < first.size() ? first[i] : second[i - first.size()];
assert(region.srcOffset == i * 4);
assert(region.dstOffset == i * 8);
assert(region.size == pass + 1);
}
++executed;
}));
}
// The queued commands must use owned data after the source is destroyed.
std::fill(regions.begin(), regions.end(), vk::BufferCopy{});
regions.clear();
regions.shrink_to_fit();
chunk.Execute({});
assert(executed == pass + 1);
assert(chunk.Empty());
assert(chunk.Size() == 0);
}
// A new spill or a return to inline storage must keep all older spills alive.
assert(chunk.ReserveData(RecordChunk::Capacity * 2));
auto* first = static_cast<unsigned char*>(chunk.Allocate(RecordChunk::Capacity * 2, 1));
assert(first);
std::memset(first, 0x5a, RecordChunk::Capacity * 2);
assert(chunk.ReserveData(RecordChunk::Capacity * 3));
auto* second = static_cast<unsigned char*>(chunk.Allocate(RecordChunk::Capacity * 3, 1));
assert(second);
std::memset(second, 0xa5, RecordChunk::Capacity * 3);
assert(chunk.ReserveData(32));
assert(chunk.Allocate(32, 1));
assert(chunk.Push([first, second](vk::CommandBuffer) {
for (size_t i = 0; i < RecordChunk::Capacity * 2; ++i) {
assert(first[i] == 0x5a);
}
for (size_t i = 0; i < RecordChunk::Capacity * 3; ++i) {
assert(second[i] == 0xa5);
}
}));
chunk.Execute({});
// Spilling requires command room; a full chunk must be retired and retried.
unsigned queued = 0;
unsigned consumed = 0;
while (chunk.Push([&consumed](vk::CommandBuffer) { ++consumed; })) {
++queued;
}
const auto full_size = chunk.Size();
assert(!chunk.ReserveData(RecordChunk::Capacity * 2));
assert(chunk.Size() == full_size);
chunk.Execute({});
assert(consumed == queued);
assert(chunk.ReserveData(RecordChunk::Capacity * 2));
chunk.Execute({});
// The exact inline limit leaves room for its command, and size checks cannot wrap.
assert(chunk.ReserveData(RecordChunk::Capacity - RecordChunk::CommandReserve));
assert(chunk.Allocate(RecordChunk::Capacity - RecordChunk::CommandReserve, 1));
assert(!chunk.Allocate(std::numeric_limits<size_t>::max(), 1));
assert(chunk.Push([](vk::CommandBuffer) {}));
chunk.Execute({});
std::puts(
"PASS: record data ownership, grouped alignment, multiple spills and chunk recycling");
}
+138
View File
@@ -0,0 +1,138 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <cassert>
#include <cstring>
#include <limits>
#include <string>
#include <vector>
#include "common/serdes.h"
namespace {
template <typename F>
void ExpectCorrupt(F&& read) {
bool caught = false;
try {
read();
} catch (const Serialization::CorruptData&) {
caught = true;
}
assert(caught);
}
template <typename T>
void Append(std::vector<u8>& bytes, const T& value) {
const auto* first = reinterpret_cast<const u8*>(&value);
bytes.insert(bytes.end(), first, first + sizeof(value));
}
void RoundTripAndTruncation() {
const u32 version = 5;
const std::string text = "shader";
const std::vector<u32> words{0x12345678, 0x9abcdef0};
Serialization::Archive written;
Serialization::Writer writer{written};
writer.Write(version);
writer.Write(text);
writer.Write(words);
auto bytes = written.TakeOff();
// The on-disk layout stays native scalar/count/payload, with no new fields.
std::vector<u8> expected;
Append(expected, version);
Append(expected, text.size());
expected.insert(expected.end(), text.begin(), text.end());
Append(expected, words.size());
for (const auto word : words) {
Append(expected, word);
}
assert(bytes == expected);
const auto read = [&](std::vector<u8> record) {
Serialization::Archive ar{std::move(record)};
Serialization::Reader reader{ar};
u32 got_version{};
std::string got_text;
std::vector<u32> got_words;
reader.Read(got_version);
reader.Read(got_text);
reader.Read(got_words);
assert(got_version == version && got_text == text && got_words == words);
assert(ar.Remaining() == 0 && ar.IsEoS());
};
read(bytes);
for (size_t length = 0; length < bytes.size(); ++length) {
ExpectCorrupt([&] { read({bytes.begin(), bytes.begin() + length}); });
}
}
void InvalidLengths() {
for (const size_t count : {size_t{2}, std::numeric_limits<size_t>::max()}) {
std::vector<u8> bytes;
Append(bytes, count);
bytes.push_back(0);
{
Serialization::Archive ar{std::vector<u8>{bytes}};
Serialization::Reader reader{ar};
std::string text = "unchanged";
ExpectCorrupt([&] { reader.Read(text); });
assert(text == "unchanged" && ar.Remaining() == 1);
}
{
Serialization::Archive ar{std::vector<u8>{bytes}};
Serialization::Reader reader{ar};
std::vector<u8> values;
ExpectCorrupt([&] { reader.Read(values); });
assert(values.empty() && ar.Remaining() == 1);
}
}
// A count may fit the byte bound while the last multi-byte element is truncated.
std::vector<u8> bytes;
Append(bytes, size_t{1});
bytes.push_back(0);
Serialization::Archive ar{std::move(bytes)};
Serialization::Reader reader{ar};
std::vector<u64> values;
ExpectCorrupt([&] { reader.Read(values); });
assert(ar.Remaining() == 1);
}
void EmptyContainers() {
Serialization::Archive ar;
Serialization::Writer writer{ar};
writer.Write(std::string{});
writer.Write(std::vector<u32>{});
auto bytes = ar.TakeOff();
Serialization::Archive input{std::move(bytes)};
Serialization::Reader reader{input};
std::string text = "old";
std::vector<u32> values;
reader.Read(text);
reader.Read(values);
assert(text.empty() && values.empty() && input.Remaining() == 0);
}
void InvalidByteRanges() {
Serialization::Archive ar{std::vector<u8>{1, 2, 3}};
ar.Advance(1);
ExpectCorrupt([&] { ar.Advance(std::numeric_limits<size_t>::max()); });
assert(ar.Remaining() == 2);
Serialization::Reader reader{ar};
u32 value = 0xdeadbeef;
ExpectCorrupt([&] { reader.Read(value); });
assert(value == 0xdeadbeef && ar.Remaining() == 2);
ExpectCorrupt([&] { reader.Read(&value, std::numeric_limits<size_t>::max()); });
assert(value == 0xdeadbeef && ar.Remaining() == 2);
ar.Advance(2);
assert(ar.IsEoS());
ExpectCorrupt([&] { ar.Advance(1); });
assert(ar.Remaining() == 0);
}
} // namespace
int main() {
RoundTripAndTruncation();
InvalidLengths();
EmptyContainers();
InvalidByteRanges();
}
+141
View File
@@ -0,0 +1,141 @@
// SPDX-License-Identifier: GPL-2.0-or-later
// Uses the production descriptor reader; no game files, Vulkan device or window needed.
#include <array>
#include <cassert>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <limits>
#include "shader_recompiler/frontend/fetch_shader.h"
// Match async-fetch-snapshot-test: link the production fetch reader without a GPU runtime.
extern "C" {
u32 runtime_disabled_optimizations = 0;
}
void assert_fail_impl() {
std::abort();
}
[[noreturn]] void unreachable_impl() {
std::abort();
}
namespace Common::Log {
bool ShouldLog(Level) {
return false;
}
void Write(std::string_view, Level, const char*, int, const char*, std::string&&) {}
} // namespace Common::Log
int main() {
constexpr u32 pointer_reg = 2;
constexpr u32 descriptor_offset = 4;
std::array<u32, Shader::NUM_USER_DATA_REGS> registers{};
std::array<u32, 8> table{};
Shader::Info info{};
info.user_data = registers;
AmdGpu::Buffer expected{};
expected.base_address = 0x123400;
expected.stride = 16;
expected.num_records = 12;
std::memcpy(table.data() + descriptor_offset, &expected, sizeof(expected));
const auto set_pointer = [&](u64 pointer) {
std::memcpy(registers.data() + pointer_reg, &pointer, sizeof(pointer));
};
const auto read = [&] {
return info.ReadUdReg<AmdGpu::Buffer>(pointer_reg, descriptor_offset);
};
// Regression: null table plus a nonzero descriptor offset must not read low memory.
assert(read() == AmdGpu::Buffer{});
assert(!read());
// Pointer metadata is masked before deciding whether the table is null.
set_pointer(0xabcd000000000000ULL);
assert(read() == AmdGpu::Buffer{});
assert(info.UdPointerNull(pointer_reg));
// Low addresses stay empty after masking, but the first valid address is not null.
for (const u64 low : {0x8ULL, 0xffffULL, 0xabcd00000000ffffULL}) {
set_pointer(low);
assert(info.UdPointerNull(pointer_reg));
assert(read() == AmdGpu::Buffer{});
}
set_pointer(0x10000);
assert(!info.UdPointerNull(pointer_reg));
assert(!info.UdPointerNull(Shader::IR::NumScalarRegs));
set_pointer(0);
// The vertex consumer must still see an empty descriptor with load overrides applied.
Shader::Gcn::VertexAttribute attribute{};
attribute.sgpr_base = pointer_reg;
attribute.dword_offset = descriptor_offset;
attribute.inst_offset = 8;
attribute.data_format = static_cast<u8>(AmdGpu::DataFormat::Format32);
assert(!attribute.GetSharp(info));
// Valid tables, masked pointer metadata and instruction offsets retain their behavior.
set_pointer(reinterpret_cast<u64>(table.data()) | 0xabcd000000000000ULL);
assert(read() == expected);
assert(!info.UdPointerNull(pointer_reg));
const auto vertex = attribute.GetSharp(info);
assert(vertex.num_records == expected.num_records);
assert(vertex.base_address == expected.base_address + attribute.inst_offset);
assert(vertex.data_format == attribute.data_format);
// The direct-register path bypasses the pointer indirection.
std::memcpy(registers.data() + descriptor_offset, &expected, sizeof(expected));
assert(info.ReadUdReg<AmdGpu::Buffer>(Shader::IR::NumScalarRegs, descriptor_offset) ==
expected);
// Direct reads must fit entirely, including their last dword.
constexpr u32 last_descriptor = Shader::NUM_USER_DATA_REGS - sizeof(expected) / sizeof(u32);
std::memcpy(registers.data() + last_descriptor, &expected, sizeof(expected));
assert(info.ReadUdReg<AmdGpu::Buffer>(Shader::IR::NumScalarRegs, last_descriptor) == expected);
for (const u32 offset : {last_descriptor + 1, Shader::NUM_USER_DATA_REGS,
Shader::NUM_USER_DATA_REGS + 1, std::numeric_limits<u32>::max()}) {
assert(info.ReadUdReg<AmdGpu::Buffer>(Shader::IR::NumScalarRegs, offset) ==
AmdGpu::Buffer{});
}
// Missing and truncated pointer pairs must be rejected by both consumers.
for (const u32 index : {Shader::NUM_USER_DATA_REGS - 1, Shader::NUM_USER_DATA_REGS,
Shader::NUM_USER_DATA_REGS + 1, std::numeric_limits<u32>::max()}) {
assert(info.UdPointerNull(index));
assert(info.ReadUdReg<AmdGpu::Buffer>(index, 0) == AmdGpu::Buffer{});
}
// The final complete pointer pair remains usable.
constexpr u32 last_pointer = Shader::NUM_USER_DATA_REGS - 2;
const auto table_pointer = reinterpret_cast<u64>(table.data());
std::memcpy(registers.data() + last_pointer, &table_pointer, sizeof(table_pointer));
assert(!info.UdPointerNull(last_pointer));
assert(info.ReadUdReg<AmdGpu::Buffer>(last_pointer, descriptor_offset) == expected);
info.user_data = {};
assert(info.UdPointerNull(pointer_reg));
assert(read() == AmdGpu::Buffer{});
assert(info.ReadUdReg<AmdGpu::Buffer>(Shader::IR::NumScalarRegs, 0) == AmdGpu::Buffer{});
info.user_data = registers;
// Draw snapshots take precedence over current user data, including a null table.
std::array<u32, Shader::NUM_USER_DATA_REGS> snapshot_registers{};
Shader::Info::ud_snapshots[0] = {
&info, snapshot_registers.data(), static_cast<u32>(snapshot_registers.size()), nullptr, 0,
0,
};
Shader::Info::num_ud_snapshots = 1;
assert(read() == AmdGpu::Buffer{});
assert(!attribute.GetSharp(info));
assert(info.UdPointerNull(pointer_reg));
const auto snapshot_pointer = reinterpret_cast<u64>(table.data());
std::memcpy(snapshot_registers.data() + pointer_reg, &snapshot_pointer,
sizeof(snapshot_pointer));
assert(read() == expected);
assert(!info.UdPointerNull(pointer_reg));
Shader::Info::ud_snapshots[0].user_data_size = pointer_reg + 1;
assert(info.UdPointerNull(pointer_reg));
assert(read() == AmdGpu::Buffer{});
assert(info.ReadUdReg<AmdGpu::Buffer>(Shader::IR::NumScalarRegs, 0) == AmdGpu::Buffer{});
Shader::Info::num_ud_snapshots = 0;
std::puts("Shader user data: null/short tables, vertex consumers, direct registers and "
"snapshots passed");
}
+114
View File
@@ -0,0 +1,114 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <array>
#include <cassert>
#include <cstdio>
#include <cstdint>
#include <cstdlib>
#include <type_traits>
#include <vector>
#include "common/slot_vector.h"
// This standalone test uses the production header and page operations, with a fatal
// assertion sink instead of the GPU logger. It needs no renderer or game resources.
void assert_fail_impl() {
std::abort();
}
namespace Common::Log {
bool ShouldLog(Level) {
return true;
}
void Write(std::string_view, Level, const char*, int, const char*, std::string&& message) {
std::fprintf(stderr, "%s\n", message.c_str());
}
} // namespace Common::Log
struct alignas(64) Item {
explicit Item(unsigned value_) : value(value_) {
++constructed;
++live;
payload.fill(value);
}
Item(const Item&) = delete;
Item(Item&&) = delete;
~Item() {
--live;
++destroyed;
}
unsigned value;
std::array<unsigned, 38> payload; // 192-byte entries straddle 2 MiB chunk boundaries.
static inline unsigned live = 0, constructed = 0, destroyed = 0;
};
int main() {
constexpr unsigned count = 50000;
static_assert(sizeof(Item) == 192);
static_assert(!std::is_copy_constructible_v<Common::SlotVector<Item>>);
static_assert(!std::is_move_constructible_v<Common::SlotVector<Item>>);
std::vector<Common::SlotId> ids;
std::vector<Item*> addresses;
ids.reserve(count);
addresses.reserve(count);
{
Common::SlotVector<Item> pool;
assert(pool.size() == 0);
assert(pool.begin() == pool.end());
// The first 2 MiB chunk rounds its allocation bitset up to a full word.
constexpr u32 initial_capacity = (2u << 20) / sizeof(Item);
constexpr u32 bitset_end = ((initial_capacity + 63) / 64) * 64;
assert(!pool.is_allocated(Common::SlotId{0}));
assert(!pool.is_allocated(Common::SlotId{63}));
assert(!pool.is_allocated(Common::SlotId{64}));
assert(!pool.is_allocated(Common::SlotId{bitset_end - 1}));
assert(!pool.is_allocated(Common::SlotId{bitset_end}));
assert(!pool.is_allocated(Common::SlotId{}));
assert(!pool.is_allocated(Common::SlotId{Common::SlotId::INVALID_INDEX - 1}));
for (unsigned i = 0; i < count; ++i) {
ids.push_back(pool.insert(i));
assert(pool.is_allocated(ids.back()));
addresses.push_back(&pool[ids.back()]);
assert(reinterpret_cast<std::uintptr_t>(addresses.back()) % alignof(Item) == 0);
assert(&pool[ids.front()] == addresses.front());
}
assert(pool.size() == count);
assert(Item::live == count);
assert(!pool.is_allocated(Common::SlotId{}));
assert(!pool.is_allocated(Common::SlotId{Common::SlotId::INVALID_INDEX - 1}));
unsigned visited = 0;
for (auto& item : pool) {
assert(item.value == visited);
assert(&item == addresses[visited]);
for (auto value : item.payload) {
assert(value == visited);
}
++visited;
}
assert(visited == count);
for (unsigned i = 0; i < count; i += 2) {
pool.erase(ids[i]);
assert(!pool.is_allocated(ids[i]));
}
assert(pool.size() == count / 2);
visited = 0;
for (auto& item : pool) {
assert(item.value == visited * 2 + 1);
++visited;
}
assert(visited == count / 2);
for (unsigned i = 0; i < count / 2; ++i) {
const auto id = pool.insert(count + i);
assert(pool.is_allocated(id));
assert(id.index % 2 == 0);
assert(&pool[id] == addresses[id.index]);
}
for (unsigned i = 1; i < count; i += 2) {
assert(&pool[ids[i]] == addresses[i]);
assert(pool[ids[i]].value == i);
}
assert(pool.size() == count);
}
assert(Item::live == 0);
assert(Item::constructed == count + count / 2);
assert(Item::destroyed == Item::constructed);
std::puts("SlotVector: 50000 stable entries, alignment, iteration, reuse and destruction passed");
}
+76
View File
@@ -0,0 +1,76 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <cassert>
#include <cstdio>
#include <limits>
#include "video_core/texture_cache/types.h"
using VideoCore::SubresourceExtent;
using VideoCore::SubresourceRange;
static void TestViewRangeClamp() {
struct Case {
SubresourceExtent resources;
SubresourceRange requested;
SubresourceRange expected;
};
constexpr u32 max = std::numeric_limits<u32>::max();
const Case cases[] = {
{{4, 6}, {{1, 2}, {2, 3}}, {{1, 2}, {2, 3}}}, // Valid subview is unchanged.
{{4, 6}, {{1, 2}, {3, 4}}, {{1, 2}, {3, 4}}}, // Exact end is valid.
{{4, 6}, {{2, 1}, {4, 3}}, {{2, 1}, {2, 3}}}, // Mip-only overflow preserves layers.
{{4, 6}, {{1, 4}, {2, 6}}, {{1, 4}, {2, 2}}}, // Layer-only overflow preserves mips.
{{4, 6}, {{2, 4}, {4, 6}}, {{2, 4}, {2, 2}}}, // Overlap offsets shorten both counts.
{{4, 6}, {{4, 6}, {2, 3}}, {{3, 5}, {1, 1}}}, // Base at the end moves inside.
{{4, 6}, {{8, 9}, {2, 3}}, {{3, 5}, {1, 1}}},
{{4, 6}, {{1, 2}, {0, 0}}, {{1, 2}, {1, 1}}}, // Empty views need positive counts.
{{4, 6}, {{1, 2}, {0, 3}}, {{1, 2}, {1, 3}}},
{{4, 6}, {{1, 2}, {2, 0}}, {{1, 2}, {2, 1}}},
{{4, 6}, {{1, 2}, {max, max}}, {{1, 2}, {3, 4}}}, // Addition would wrap.
{{4, 6}, {{max, max}, {1, 1}}, {{3, 5}, {1, 1}}},
{{0, 0}, {{9, 9}, {3, 3}}, {{0, 0}, {1, 1}}}, // Retain the source's minimum size.
{{max, max}, {{0, 0}, {max, max}}, {{0, 0}, {max, max}}},
{{max, max}, {{max - 1, max - 1}, {2, 2}}, {{max - 1, max - 1}, {1, 1}}},
};
static_assert(noexcept(SubresourceRange{}.ClampTo(SubresourceExtent{})));
for (const auto& test : cases) {
auto range = test.requested;
assert(range.ClampTo(test.resources) == (test.requested != test.expected));
assert(range == test.expected);
assert(!range.ClampTo(test.resources)); // Repeated cache hits are stable.
}
}
int main() {
TestViewRangeClamp();
struct Case {
SubresourceExtent cached;
SubresourceExtent requested;
bool contains;
};
constexpr u32 max = std::numeric_limits<u32>::max();
const Case cases[] = {
{{1, 1}, {1, 1}, true},
{{4, 2}, {3, 1}, true},
{{4, 1}, {3, 2}, false}, // More mips cannot cover missing layers.
{{3, 2}, {4, 1}, false}, // More layers cannot cover missing mips.
{{3, 1}, {3, 2}, false},
{{3, 2}, {3, 1}, true},
{{3, 2}, {4, 2}, false},
{{4, 2}, {3, 2}, true},
{{2, 1}, {3, 2}, false},
{{0, 0}, {0, 0}, true},
{{max, max}, {max, max}, true},
{{max, max - 1}, {max - 1, max}, false},
{{max - 1, max}, {max, max - 1}, false},
};
static_assert(noexcept(SubresourceExtent{}.Contains(SubresourceExtent{})));
for (const auto& test : cases) {
assert(test.cached.Contains(test.requested) == test.contains);
assert(test.cached.Contains(test.cached));
}
const SubresourceExtent extent{3, 2};
assert((extent == SubresourceExtent{3, 2}));
assert((extent != SubresourceExtent{3, 1}));
std::puts("SubresourceExtent: containment, equality and view range clamping passed");
}
+31 -5
View File
@@ -10,6 +10,15 @@
using namespace Vulkan;
struct TestPipeline {
int id;
bool valid = true;
bool IsValid() const {
return valid;
}
};
int main() {
#ifdef _WIN32
_set_error_mode(_OUT_TO_STDERR);
@@ -54,13 +63,30 @@ int main() {
assert(inbox.Take(8).empty());
// Conflict: the first pipeline for a key wins, the later one is left to the caller.
std::map<int, std::unique_ptr<int>> pipelines;
auto runtime = std::make_unique<int>(1);
auto warm = std::make_unique<int>(2);
std::map<int, std::unique_ptr<TestPipeline>> pipelines;
auto runtime = std::make_unique<TestPipeline>(1);
auto warm = std::make_unique<TestPipeline>(2);
assert(PublishFirst(pipelines, 7, std::move(runtime)));
assert(!PublishFirst(pipelines, 7, std::move(warm)));
assert(*pipelines[7] == 1 && warm && *warm == 2);
assert(PublishFirst(pipelines, 8, std::make_unique<int>(3)) && *pipelines[8] == 3);
assert(pipelines[7]->id == 1 && warm && warm->id == 2);
assert(PublishFirst(pipelines, 8, std::make_unique<TestPipeline>(3)) && pipelines[8]->id == 3);
// A refused build cannot occupy a slot or prevent a later valid pipeline from winning.
auto refused = std::make_unique<TestPipeline>(4, false);
assert(!PublishFirst(pipelines, 9, std::move(refused)));
assert(!pipelines.contains(9) && refused && !refused->IsValid());
assert(!PublishFirst(pipelines, 9, std::unique_ptr<TestPipeline>{}));
assert(!pipelines.contains(9));
assert(PublishFirst(pipelines, 9, std::make_unique<TestPipeline>(5)));
assert(pipelines[9]->id == 5);
assert(!PublishFirst(pipelines, 9, std::move(refused)));
assert(pipelines[9]->id == 5 && refused);
// A runtime refusal marker suppresses future warm-up publication and per-draw retries.
pipelines.try_emplace(10);
auto retry = std::make_unique<TestPipeline>(6);
assert(!PublishFirst(pipelines, 10, std::move(retry)));
assert(!pipelines[10] && retry && retry->id == 6);
std::puts("PASS: warm-up inbox");
}