#include "VulkanShaderCompiler.h" #include "VulkanShader.h" #include "def_files/def_files.h" #include "external_dll/externalcode.h" #include "graphics/shader_preprocess.h" #include "graphics/shader_types.h" #include "globalincs/systemvars.h" #include "graphics/post_processing.h" #include #include #include namespace graphics::vulkan { // ========== VulkanShadercLibrary ========== class VulkanShadercLibrary : public SCP_ExternalCode { public: VulkanShadercLibrary(); bool isLoaded() const { return m_loaded; } decltype(&shaderc_compiler_initialize) compiler_initialize = nullptr; decltype(&shaderc_compiler_release) compiler_release = nullptr; decltype(&shaderc_compile_options_initialize) compile_options_initialize = nullptr; decltype(&shaderc_compile_options_release) compile_options_release = nullptr; decltype(&shaderc_compile_options_set_target_env) compile_options_set_target_env = nullptr; decltype(&shaderc_compile_options_set_optimization_level) compile_options_set_optimization_level = nullptr; decltype(&shaderc_compile_options_set_generate_debug_info) compile_options_set_generate_debug_info = nullptr; decltype(&shaderc_compile_into_spv) compile_into_spv = nullptr; decltype(&shaderc_result_release) result_release = nullptr; decltype(&shaderc_result_get_compilation_status) result_get_compilation_status = nullptr; decltype(&shaderc_result_get_error_message) result_get_error_message = nullptr; decltype(&shaderc_result_get_num_warnings) result_get_num_warnings = nullptr; decltype(&shaderc_result_get_bytes) result_get_bytes = nullptr; decltype(&shaderc_result_get_length) result_get_length = nullptr; private: bool m_loaded = false; }; VulkanShadercLibrary::VulkanShadercLibrary() { // Try platform-specific library names #if defined(_WIN32) const char* names[] = {"shaderc_shared.dll", "shaderc.dll"}; #elif defined(__APPLE__) const char* names[] = {"libshaderc_shared.1.dylib", "libshaderc_shared.dylib", "libshaderc.dylib"}; #else const char* names[] = {"libshaderc_shared.so.1", "libshaderc_shared.so", "libshaderc.so.1", "libshaderc.so"}; #endif auto base_path = SDL_GetBasePath(); bool loaded = false; for (const auto* name : names) { if (LoadExternal(name, base_path)) { loaded = true; nprintf(("vulkan", "VulkanShadercLibrary: Loaded '%s'\n", name)); break; } } if (!loaded) { nprintf(("vulkan", "VulkanShadercLibrary: Could not load shaderc shared library\n")); return; } // Load all required function pointers compiler_initialize = LoadFunction("shaderc_compiler_initialize"); compiler_release = LoadFunction("shaderc_compiler_release"); compile_options_initialize = LoadFunction("shaderc_compile_options_initialize"); compile_options_release = LoadFunction("shaderc_compile_options_release"); compile_options_set_target_env = LoadFunction("shaderc_compile_options_set_target_env"); compile_options_set_optimization_level = LoadFunction("shaderc_compile_options_set_optimization_level"); compile_options_set_generate_debug_info = LoadFunction("shaderc_compile_options_set_generate_debug_info"); compile_into_spv = LoadFunction("shaderc_compile_into_spv"); result_release = LoadFunction("shaderc_result_release"); result_get_compilation_status = LoadFunction("shaderc_result_get_compilation_status"); result_get_error_message = LoadFunction("shaderc_result_get_error_message"); result_get_num_warnings = LoadFunction("shaderc_result_get_num_warnings"); result_get_bytes = LoadFunction("shaderc_result_get_bytes"); result_get_length = LoadFunction("shaderc_result_get_length"); // Verify all required functions were loaded m_loaded = compiler_initialize && compiler_release && compile_options_initialize && compile_options_release && compile_options_set_target_env && compile_options_set_optimization_level && compile_options_set_generate_debug_info && compile_into_spv && result_release && result_get_compilation_status && result_get_error_message && result_get_num_warnings && result_get_bytes && result_get_length; if (!m_loaded) { nprintf(("vulkan", "VulkanShadercLibrary: Library loaded but some functions are missing!\n")); } } // ========== VulkanShaderCompiler ========== VulkanShaderCompiler::VulkanShaderCompiler() = default; VulkanShaderCompiler::~VulkanShaderCompiler() = default; bool VulkanShaderCompiler::init(bool hwClipDistance) { m_hwClipDistance = hwClipDistance; if (m_initialized) { return true; } m_shaderc = std::make_unique(); if (!m_shaderc->isLoaded()) { nprintf(("vulkan", "VulkanShaderCompiler: shaderc library not available!\n" " Install the Vulkan SDK or shaderc shared library:\n" " Debian/Ubuntu: sudo apt install libshaderc-dev\n" " Vulkan SDK: https://vulkan.lunarg.com/sdk/home\n")); m_shaderc.reset(); return false; } m_initialized = true; nprintf(("vulkan", "VulkanShaderCompiler: Initialized (runtime shaderc compilation)\n")); return true; } void VulkanShaderCompiler::shutdown() { m_shaderc.reset(); m_initialized = false; } bool VulkanShaderCompiler::isAvailable() const { return m_initialized && m_shaderc && m_shaderc->isLoaded(); } SCP_string VulkanShaderCompiler::buildHeader(vk::ShaderStageFlagBits /*stage*/, shader_type sdrType, unsigned int flags) const { (void)this; SCP_string header; header.reserve(512); // Required for layout(location=N) on varyings in Vulkan GLSL. // Injected here so individual shaders don't need it. header += "#extension GL_ARB_separate_shader_objects : enable\n"; // shaderc automatically predefines VULKAN=100 when targeting Vulkan, // so we do NOT define it here — doing so causes a "Macro redefined" error. // gl_ClipDistance[] requires the shaderClipDistance device feature. When the // device supports it, use the hardware clip path, otherwise // the shaders fall back to the vClipDist varying (software clipping). if (!m_hwClipDistance) { header += "#define SDR_FLAG_SW_CLIP_FALLBACK\n"; } // Blinn-Phong lighting model (matches OpenGL's opengl_shader_get_header) if (Detail.lighting < 3) { header += "#define FLAG_LIGHT_MODEL_BLINN_PHONG\n"; } header += shader_get_shadow_cascade_defines(); header += shader_get_rt_shadow_light_limit_define(); // Post-processing shaders need special header injection (matching OpenGL's // opengl_post_shader_header). Effect indices map to #define names, and // lightshafts needs the sample count. if (sdrType == SDR_TYPE_POST_PROCESS_MAIN || sdrType == SDR_TYPE_POST_PROCESS_LIGHTSHAFTS) { if (graphics::Post_processing_manager) { if (sdrType == SDR_TYPE_POST_PROCESS_MAIN) { const auto& postEffects = graphics::Post_processing_manager->getPostEffects(); for (size_t idx = 0; idx < postEffects.size(); idx++) { if (flags & (1 << idx)) { header += "#define "; header += postEffects[idx].define_name; header += "\n"; } } } else if (sdrType == SDR_TYPE_POST_PROCESS_LIGHTSHAFTS) { const auto& ls_params = graphics::Post_processing_manager->getLightshaftParams(); char temp[64]; snprintf(temp, sizeof(temp), "#define SAMPLE_NUM %d\n", ls_params.samplenum); header += temp; } } } else if (gr_is_fxaa_mode(Gr_aa_mode) && sdrType == SDR_TYPE_POST_PROCESS_FXAA) { // GLSL 450 always has textureGather header += shader_get_fxaa_defines(Gr_aa_mode, true); } else if (gr_is_smaa_mode(Gr_aa_mode) && (sdrType == SDR_TYPE_POST_PROCESS_SMAA_EDGE || sdrType == SDR_TYPE_POST_PROCESS_SMAA_BLENDING_WEIGHT || sdrType == SDR_TYPE_POST_PROCESS_SMAA_NEIGHBORHOOD_BLENDING)) { // GLSL 450 always has textureGather header += shader_get_smaa_defines(Gr_aa_mode, true); } else { // Inject variant-specific #defines based on flags header += shader_build_variant_defines(sdrType, flags); } return header; } SCP_string VulkanShaderCompiler::computeSourceHash(const SCP_string& header, const SCP_string& source) { MD5 md5; md5.update(header.c_str(), static_cast(header.size())); md5.update(source.c_str(), static_cast(source.size())); // Include a version tag so cache is invalidated on engine updates static const char VERSION_TAG[] = "vk_shader_v1"; md5.update(VERSION_TAG, sizeof(VERSION_TAG) - 1); md5.finalize(); return md5.hexdigest(); } // rejects a cache file cut short by a crash or full disk: a complete module ends exactly on an OpFunctionEnd static bool isCompleteSpirv(const SCP_vector& spirv) { constexpr uint32_t SPIRV_MAGIC = 0x07230203; constexpr size_t HEADER_WORDS = 5; constexpr uint32_t OP_FUNCTION_END = 56; if (spirv.size() < HEADER_WORDS || spirv[0] != SPIRV_MAGIC) { return false; } size_t i = HEADER_WORDS; uint32_t lastOpcode = 0; while (i < spirv.size()) { const uint32_t wordCount = spirv[i] >> 16; if (wordCount == 0) { return false; } lastOpcode = spirv[i] & 0xFFFF; i += wordCount; } return i == spirv.size() && lastOpcode == OP_FUNCTION_END; } SCP_vector VulkanShaderCompiler::compile(const SCP_string& filename, vk::ShaderStageFlagBits stage, shader_type sdrType, unsigned int flags, bool requiresRaytracing) { if (!m_initialized || !m_shaderc) { nprintf(("vulkan", "VulkanShaderCompiler: Not initialized!\n")); return {}; } // Load and preprocess GLSL source SCP_string source = shader_load_source(filename); if (source.empty()) { nprintf(("vulkan", "VulkanShaderCompiler: Failed to load GLSL source: %s\n", filename.c_str())); return {}; } source = shader_preprocess_includes(filename, source); source = shader_preprocess_defines(filename, source); // Build preprocessor header with #defines for variant flags SCP_string header = buildHeader(stage, sdrType, flags); // Compute hash for disk cache (uses preprocessed source) SCP_string hash = computeSourceHash(header, source); SCP_string cacheFilename = "vk_shader-" + hash + ".spv"; // Check disk cache CFILE* cacheFile = cfopen(cacheFilename.c_str(), "rb", CF_TYPE_CACHE); if (cacheFile != nullptr) { int fileSize = cfilelength(cacheFile); if (fileSize > 0 && (fileSize % 4) == 0) { SCP_vector spirv(fileSize / 4); if (cfread(spirv.data(), 1, fileSize, cacheFile) == fileSize) { if (isCompleteSpirv(spirv)) { cfclose(cacheFile); nprintf(("vulkan", "VulkanShaderCompiler: Cache hit for %s (flags=0x%x)\n", filename.c_str(), flags)); return spirv; } mprintf(("VulkanShaderCompiler: Ignoring incomplete cache file %s\n", cacheFilename.c_str())); } } cfclose(cacheFile); } // Cache miss — compile with shaderc nprintf(("vulkan", "VulkanShaderCompiler: Compiling %s (flags=0x%x)...\n", filename.c_str(), flags)); // Assemble: #version + header (extension + defines) + source SCP_string fullSource; fullSource.reserve(header.size() + source.size() + 32); // glslang only populates the GL_EXT_ray_query builtin symbol table (rayQueryEXT, // gl_RayFlags*EXT, etc.) under GLSL 460 -- declaring the extension alone under // #version 450 compiles but leaves those identifiers undeclared. Scoped to the // same requiresRaytracing variants as the target-env bump above. fullSource += requiresRaytracing ? "#version 460\n" : "#version 450\n"; fullSource += header; fullSource += source; auto* sc = m_shaderc.get(); shaderc_compiler_t compiler = sc->compiler_initialize(); if (!compiler) { nprintf(("vulkan", "VulkanShaderCompiler: Failed to initialize shaderc compiler!\n")); return {}; } shaderc_compile_options_t opts = sc->compile_options_initialize(); // Only bump the target-env for the specific shader variants that actually use // GL_EXT_ray_query (see shader_variant_requires_raytracing()) -- everything // else keeps targeting the lowest common Vulkan version so a higher SPIR-V // version tag doesn't risk rejection by more conservative Vulkan 1.0/1.1 ICDs. sc->compile_options_set_target_env(opts, shaderc_target_env_vulkan, requiresRaytracing ? shaderc_env_version_vulkan_1_2 : shaderc_env_version_vulkan_1_0); sc->compile_options_set_optimization_level(opts, shaderc_optimization_level_performance); sc->compile_options_set_generate_debug_info(opts); shaderc_shader_kind kind; if (stage == vk::ShaderStageFlagBits::eVertex) { kind = shaderc_vertex_shader; } else { kind = shaderc_fragment_shader; } shaderc_compilation_result_t result = sc->compile_into_spv( compiler, fullSource.c_str(), fullSource.size(), kind, filename.c_str(), "main", opts); SCP_vector spirv; auto status = sc->result_get_compilation_status(result); if (status != shaderc_compilation_status_success) { const char* errMsg = sc->result_get_error_message(result); nprintf(("vulkan", "VulkanShaderCompiler: COMPILATION FAILED for %s (flags=0x%x):\n%s\n", filename.c_str(), flags, errMsg ? errMsg : "(no error message)")); } else { if (sc->result_get_num_warnings(result) > 0) { const char* errMsg = sc->result_get_error_message(result); nprintf(("vulkan", "VulkanShaderCompiler: Warnings for %s:\n%s\n", filename.c_str(), errMsg ? errMsg : "")); } size_t byteLen = sc->result_get_length(result); const char* bytes = sc->result_get_bytes(result); if (bytes && byteLen > 0 && (byteLen % 4) == 0) { spirv.resize(byteLen / 4); std::memcpy(spirv.data(), bytes, byteLen); nprintf(("vulkan", "VulkanShaderCompiler: Compiled %s -> %zu bytes SPIR-V\n", filename.c_str(), byteLen)); // each writer renames its own temp file, so a game crash or a second game never leaves a truncated .spv const SCP_string tmpFilename = "vk_shader-" + hash + "-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()) + ".tmp"; const bool complete = isCompleteSpirv(spirv); cacheFile = complete ? cfopen(tmpFilename.c_str(), "wb", CF_TYPE_CACHE) : nullptr; if (!complete) { mprintf(("VulkanShaderCompiler: Not caching %s: compiled module fails the cache check\n", cacheFilename.c_str())); } else if (cacheFile != nullptr) { const bool written = cfwrite(spirv.data(), static_cast(spirv.size() * sizeof(uint32_t)), 1, cacheFile) == 1; const bool closed = cfclose(cacheFile) == 0; bool saved = false; if (written && closed) { // rename() on Windows fails if the target exists cf_delete(cacheFilename.c_str(), CF_TYPE_CACHE); saved = cf_rename(tmpFilename.c_str(), cacheFilename.c_str(), CF_TYPE_CACHE) == CF_RENAME_SUCCESS; } if (!saved) { cf_delete(tmpFilename.c_str(), CF_TYPE_CACHE); mprintf(("VulkanShaderCompiler: Could not write cache file %s\n", cacheFilename.c_str())); } } } } sc->result_release(result); sc->compile_options_release(opts); sc->compiler_release(compiler); return spirv; } void VulkanShaderCompiler::purgeOldCache() { const SCP_string PREFIX = "vk_shader-"; const auto TIMEOUT = 2.0 * 30.0 * 24.0 * 60.0 * 60.0; // ~2 months in seconds auto now = std::time(nullptr); for (const char* ext : {".spv", ".tmp"}) { SCP_vector cache_files; SCP_vector file_info; const SCP_string filter = SCP_string("*") + ext; cf_get_file_list(cache_files, CF_TYPE_CACHE, filter.c_str(), CF_SORT_NONE, &file_info, CF_LOCATION_ROOT_USER | CF_LOCATION_ROOT_GAME | CF_LOCATION_TYPE_ROOT); Assertion(cache_files.size() == file_info.size(), "cf_get_file_list returned different sizes for file names and file informations!"); for (size_t i = 0; i < cache_files.size(); ++i) { auto& name = cache_files[i]; if (name.compare(0, PREFIX.size(), PREFIX) != 0) { continue; // Not our cache file } auto diff = std::difftime(now, file_info[i].write_time); if (diff > TIMEOUT) { auto full_name = name + ext; cf_delete(full_name.c_str(), CF_TYPE_CACHE); } } } } } // namespace graphics::vulkan