fs2open.github.com/code/graphics/vulkan/VulkanRenderer.h

#pragma once

#include "osapi/osapi.h"

#include "VulkanMemory.h"
#include "VulkanBuffer.h"
#include "VulkanTexture.h"
#include "VulkanShader.h"
#include "VulkanDescriptorManager.h"
#include "VulkanPipeline.h"
#include "VulkanState.h"
#include "VulkanDraw.h"
#include "VulkanDeletionQueue.h"
#include "VulkanPostProcessing.h"
#include "VulkanQuery.h"
#include "VulkanRaytracing.h"
#include "VulkanRenderFrame.h"

#include <vulkan/vulkan.hpp>

namespace graphics::vulkan {

struct QueueIndex {
	// Poor mans std::optional
	bool initialized = false;
	uint32_t index = 0;
};

/**
 * @brief Viewport handling when beginning a tracked render pass
 */
enum class PassViewport {
	FlipY,  // Negative-height viewport for OpenGL-compatible Y-up NDC (scene/swap chain passes)
	NoFlip, // Standard positive viewport (post-processing blits)
	Keep,   // Leave the viewport untouched (off-screen RTs; the engine sets it via gr_set_viewport)
};

/**
 * @brief Parameters for VulkanRenderer::beginTrackedRenderPass()
 *
 * Bundles everything a render-pass begin must keep in sync: the pass/framebuffer,
 * the clear values, and the state-tracker bookkeeping (current pass, attachment
 * count, sample count, render area, viewport). Every render-pass begin on the
 * frame command buffer goes through this so no site can forget part of it.
 */
struct PassBeginDesc {
	vk::RenderPass renderPass;
	vk::Framebuffer framebuffer;
	vk::Extent2D extent;
	ArrayView<vk::ClearValue> clearValues; // may be empty for pure loadOp=eLoad passes
	uint32_t colorAttachmentCount = 1;
	vk::SampleCountFlagBits sampleCount = vk::SampleCountFlagBits::e1;
	PassViewport viewport = PassViewport::FlipY;
};

struct PhysicalDeviceValues {
	vk::PhysicalDevice device;
	vk::PhysicalDeviceProperties properties;
	vk::PhysicalDeviceFeatures features;

	// Ray tracing feature support, queried via a VkPhysicalDeviceFeatures2
	// pNext chain in pickPhysicalDevice(). These reflect only whether the
	// device *feature bit* is set -- whether raytraced shadows are actually
	// usable also requires the corresponding device extensions to be present
	// (checked separately in createLogicalDevice() against `extensions` below).
	bool accelerationStructureFeatureSupported = false;
	bool rayQueryFeatureSupported = false;
	bool bufferDeviceAddressFeatureSupported = false;

	// VK_KHR_synchronization2's feature bit. The extension itself is required
	// (RequiredDeviceExtensions), so this only guards against a device that
	// exposes the extension without the feature -- see isDeviceUnsuitable().
	bool synchronization2FeatureSupported = false;

	SCP_vector<vk::ExtensionProperties> extensions;

	vk::SurfaceCapabilitiesKHR surfaceCapabilities;
	SCP_vector<vk::SurfaceFormatKHR> surfaceFormats;
	SCP_vector<vk::PresentModeKHR> presentModes;

	SCP_vector<vk::QueueFamilyProperties> queueProperties;
	QueueIndex graphicsQueueIndex;
	QueueIndex presentQueueIndex;
};

class VulkanRenderer {
  public:
	explicit VulkanRenderer(std::unique_ptr<os::GraphicsOperations> graphicsOps);

	bool initialize();

	/**
	 * @brief Setup for a new frame - begins command buffer and render pass
	 * Called at the START of each frame before any draw calls
	 */
	void setupFrame();

	/**
	 * @brief Drop every cached pipeline at the start of the next frame; each is rebuilt on its next use
	 * Called after the shaders are recompiled, since a pipeline keeps the shader code it was built from
	 */
	void requestPipelineRebuild() { m_pipelinesStale = true; }

	/**
	 * @brief End frame - ends render pass, submits, and presents
	 * Called at the END of each frame after all draw calls
	 */
	void flip();

	void shutdown();

	/**
	 * @brief Read back the previous frame's framebuffer to CPU memory
	 *
	 * Copies the previously presented swap chain image to a vm_malloc'd RGBA
	 * pixel buffer. Handles the BGRA→RGBA swizzle since the swap chain uses
	 * B8G8R8A8 format. Caller must vm_free the returned buffer.
	 *
	 * @param[out] outPixels Receives the vm_malloc'd RGBA pixel buffer
	 * @param[out] outWidth  Receives the image width
	 * @param[out] outHeight Receives the image height
	 * @return true on success, false on failure
	 */
	bool readbackFramebuffer(ubyte** outPixels, uint32_t* outWidth, uint32_t* outHeight);

	/**
	 * @brief Read back the currently-bound off-screen render target's color image.
	 *
	 * Unlike readbackFramebuffer (which reads the previous frame's on-screen image),
	 * this captures the render target that is active right now via bm_set_render_target
	 * — the path gr.screenToBlob uses to grab SCPUI-generated icons. Because the target's
	 * draws live in the not-yet-submitted frame command buffer, this flushes that buffer
	 * mid-frame (submit + host wait), copies the target's color image, then resumes
	 * rendering into it with a loadOp=eLoad pass so already-drawn content survives.
	 *
	 * The target is R8G8B8A8_UNORM, so pixels come back as tightly-packed RGBA8 with
	 * alpha preserved (no swizzle, no forced opacity). Caller must vm_free the buffer.
	 *
	 * @param ts         The active render target slot (must be non-cubemap, have a framebuffer)
	 * @param[out] outPixels Receives the vm_malloc'd RGBA8 pixel buffer
	 * @param[out] outWidth  Receives the target width
	 * @param[out] outHeight Receives the target height
	 * @return true on success, false on failure
	 */
	bool readbackRenderTarget(tcache_slot_vulkan* ts, ubyte** outPixels, uint32_t* outWidth, uint32_t* outHeight);

	/**
	 * @brief Get the minimum uniform buffer offset alignment requirement
	 * @return The alignment in bytes (typically 64 or 256)
	 */
	uint32_t getMinUniformBufferOffsetAlignment() const;

	/**
	 * @brief Get the current frame number (total frames rendered)
	 */
	uint64_t getCurrentFrameNumber() const { return m_frameNumber; }

	/**
	 * @brief Wait for a specific frame's GPU work to complete
	 *
	 * Waits on that frame's fence rather than stalling the entire device.
	 *
	 * @param timeoutNs Maximum wait in nanoseconds (0 = poll)
	 * @return true if the frame is known complete. false if the timeout expired,
	 *         or if the frame has not been submitted yet (a fence taken during
	 *         the currently-recording frame cannot complete until flip();
	 *         waiting here would deadlock, so it reports "not complete").
	 */
	bool waitForFrame(uint64_t frameNumber, uint64_t timeoutNs = UINT64_MAX);

	/**
	 * @brief Wait for all GPU work to complete
	 */
	void waitIdle();

	/**
	 * @brief Get the current command buffer as a raw Vulkan handle (for ImGui)
	 */
	VkCommandBuffer getVkCurrentCommandBuffer() const;

	/**
	 * @brief Get the current frame command buffer (valid between setupFrame and flip)
	 */
	vk::CommandBuffer getCurrentCommandBuffer() const { return m_currentCommandBuffer; }

	vk::Instance getVkInstance() const { return m_vkInstance.get(); }
	vk::PhysicalDevice getPhysicalDevice() const { return m_physicalDevice; }
	vk::Device getDevice() const { return m_device.get(); }
	vk::Queue getGraphicsQueue() const { return m_graphicsQueue; }
	uint32_t getGraphicsQueueFamilyIndex() const { return m_graphicsQueueFamilyIndex; }

	/**
	 * @brief Size of the actual window / swap chain images
	 */
	vk::Extent2D getSwapChainExtent() const { return m_swapChainExtent; }

	/**
	 * @brief Size everything is rendered at, which is not always the window size
	 *
	 * The engine positions every draw in terms of gr_screen.max_w/max_h, which
	 * may differ from the window (see Cmdline_window_res and
	 * gr_window_to_render_pos). Everything upstream of the output-encode pass --
	 * composition image, depth, and the post-processor's scene and G-buffer
	 * targets -- is sized to this, and the encode pass scales it into the swap
	 * chain image. OpenGL does the same thing with Back_framebuffer.
	 */
	vk::Extent2D getRenderExtent() const { return m_renderExtent; }

	/**
	 * @brief Get the composition image the frame is currently being rendered into
	 *
	 * This is the fp16 image behind m_swapChainFramebuffers -- the whole frame
	 * (3D scene, HUD and menus alike) lands here, and encodeToSwapChain() later
	 * converts it into the actual swap chain image. Returns a null handle if
	 * composition resources do not exist yet.
	 */
	vk::Image getCurrentCompositionImage() const;

	/**
	 * @brief Check if VK_EXT_debug_utils is enabled
	 */
	bool isDebugUtilsEnabled() const { return m_debugUtilsEnabled; }

	/**
	 * @brief Get the maximum uniform buffer range
	 */
	uint32_t getMaxUniformBufferSize() const;

	/**
	 * @brief Get the maximum sampler anisotropy
	 */
	float getMaxAnisotropy() const;

	/**
	 * @brief Check if BC texture compression is supported
	 */
	bool isTextureCompressionBCSupported() const;

	/**
	 * @brief Check if depth clamping is supported (used by the shadow pass)
	 */
	bool isDepthClampSupported() const;

	/**
	 * @brief Check if vertex shader layer output is supported (for shadow cascades)
	 */
	bool supportsShaderViewportLayerOutput() const { return m_supportsShaderViewportLayerOutput; }

	/**
	 * @brief Check if raytraced shadows are supported (VK_KHR_ray_query +
	 * VK_KHR_acceleration_structure + VK_KHR_deferred_host_operations, with
	 * the corresponding feature bits enabled on the logical device)
	 */
	bool supportsRaytracedShadows() const { return m_supportsRaytracedShadows; }

	/**
	 * @brief Switch from swap chain pass to HDR scene pass
	 *
	 * Called by vulkan_scene_texture_begin(). Ends the current swap chain
	 * render pass and begins the HDR scene render pass.
	 */
	void beginSceneRendering();

	void resumeSceneRendering();

	/**
	 * @brief Switch from HDR scene pass back to swap chain
	 *
	 * Called by vulkan_scene_texture_end(). Ends the HDR scene render pass,
	 * runs post-processing, and begins the resumed swap chain render pass.
	 */
	void endSceneRendering();

	/**
	 * @brief Copy scene color to effect texture mid-scene
	 *
	 * Called by vulkan_copy_effect_texture(). Ends the current scene render
	 * pass, copies scene color → effect texture, then resumes the scene
	 * render pass with loadOp=eLoad to preserve existing content.
	 */
	void copyEffectTexture();

	/**
	 * @brief Copy scene depth mid-scene for soft particle sampling
	 *
	 * Called lazily from the first particle draw per frame. Ends the current
	 * scene render pass, copies depth → samplable copy, then resumes the
	 * scene render pass with loadOp=eLoad. No-op if already copied this frame.
	 */
	void copySceneDepthForParticles();

	/**
	 * @brief Park the scene depth so the cockpit can render on a cleared depth buffer
	 *
	 * Called by vulkan_post_process_save_zbuffer(). Ends the current scene render
	 * pass, copies scene depth → backup image, then resumes the pass with
	 * loadOp=eLoad. The caller clears the depth buffer afterwards. No-op when a
	 * save is already outstanding, or outside scene rendering.
	 */
	void saveSceneDepth();

	/**
	 * @brief Put the parked scene depth back, discarding what the cockpit wrote
	 *
	 * Called by vulkan_post_process_restore_zbuffer(). No-op unless saveSceneDepth()
	 * parked something.
	 */
	void restoreSceneDepth();

	/**
	 * @brief Check if scene depth copy is available for sampling this frame
	 */
	bool isSceneDepthCopied() const { return m_sceneDepthCopiedThisFrame; }

	/**
	 * @brief Check if we're currently rendering to the HDR scene target
	 */
	bool isSceneRendering() const { return m_sceneRendering; }

	/**
	 * @brief Begin rendering to an off-screen render target
	 *
	 * Ends the current render pass (swap chain or previous RT face) and begins
	 * a new render pass targeting the given texture's framebuffer.
	 */
	void beginRenderTarget(tcache_slot_vulkan* ts, int face);

	/**
	 * @brief End render target and resume the swap chain pass
	 *
	 * Ends the current RT render pass and resumes the swap chain render pass
	 * with loadOp=eLoad to preserve existing content.
	 *
	 * @param ts The render target just finished, or nullptr. When it has more
	 * than one mip level (e.g. the env/irradiance cubemaps), the mip chain is
	 * regenerated here via blits since nothing else populates it -- render
	 * passes only ever write to mip 0.
	 */
	void endRenderTarget(tcache_slot_vulkan* ts = nullptr);

	/**
	 * @brief Resume the swap chain render pass with loadOp=eLoad
	 *
	 * Begins a new render pass targeting the current swap chain image,
	 * preserving existing content. Used after off-screen rendering
	 * (render targets, irradiance map generation) to return to swap chain.
	 */
	void resumeSwapChainPass();

	/**
	 * @brief Begin the composition render pass afresh, clearing color and depth
	 *
	 * The loadOp=eClear counterpart to resumeSwapChainPass(): discards whatever
	 * the composition image held and starts a clean frame into the same swap
	 * chain image, without submitting anything.
	 */
	void restartCompositionPass();

	/**
	 * @brief End the in-flight render pass, if one is active
	 *
	 * Recording a transfer/build command requires being outside a render pass
	 * instance, but callers cannot generally know whether the frame currently
	 * has one open. This ends it and clears the state tracker's record of it,
	 * so a second call (or a later flip()) is a no-op rather than an error.
	 */
	void endCurrentRenderPass();

	/**
	 * @brief Resume rendering into a render target with loadOp=eLoad (preserving content)
	 *
	 * Used after a mid-frame render-target readback (readbackRenderTarget) to continue
	 * drawing into the same target without clearing what was already rendered.
	 */
	void resumeRenderTargetPass(tcache_slot_vulkan* ts);

	/**
	 * @brief Check if we're currently rendering to an off-screen render target
	 */
	bool isRenderTargetActive() const { return m_renderTargetActive; }

	/**
	 * @brief Set whether the G-buffer render pass is active
	 *
	 * Called by deferred_lighting_finish() to switch from G-buffer to
	 * scene render pass mid-frame for forward transparent rendering.
	 */
	void setUseGbufRenderPass(bool use) { m_useGbufRenderPass = use; }
	bool isUsingGbufRenderPass() const { return m_useGbufRenderPass; }

	/**
	 * @brief Get the validated MSAA sample count for deferred lighting
	 */
	vk::SampleCountFlagBits getMsaaSampleCount() const { return m_msaaSampleCount; }

	/**
	 * @brief Get the depth/stencil format chosen at device init
	 *
	 * Used by render targets created with BMP_FLAG_RENDER_TARGET_DEPTH_ATTACHMENT so
	 * their depth buffer matches the swap-chain depth format. Returns eUndefined if
	 * depth resources have not been created yet.
	 */
	vk::Format getDepthFormat() const { return m_depthFormat; }

  private:
	/**
	 * @brief Begin a render pass on the frame command buffer with full state-tracker sync
	 *
	 * Records vkCmdBeginRenderPass and updates the state tracker (render pass,
	 * color attachment count, sample count, render area, viewport) in one place.
	 */
	void beginTrackedRenderPass(const PassBeginDesc& desc);

	/**
	 * @brief Resume the scene (or G-buffer) render pass with loadOp=eLoad
	 * after a mid-scene copy (copyEffectTexture / copySceneDepthForParticles)
	 */
	void resumeScenePassAfterCopy();

	/**
	 * @brief End the scene (or G-buffer) render pass ahead of a depth-only copy
	 */
	void endScenePassForDepthCopy();

	/**
	 * @brief Restore the color attachment layouts a depth-only copy left behind,
	 * then resume the scene (or G-buffer) render pass
	 */
	void resumeScenePassAfterDepthCopy();

	bool initDisplayDevice() const;

	bool initializeInstance();

	bool initializeSurface();

	bool pickPhysicalDevice(PhysicalDeviceValues& deviceValues);

	bool createLogicalDevice(const PhysicalDeviceValues& deviceValues);

	bool createSwapChain(const PhysicalDeviceValues& deviceValues, vk::SwapchainKHR oldSwapchain = nullptr);

	void createRenderPass();

	void createFrameBuffers();

	// HDR composition + output-encode resources
	void createCompositionResources();
	void createEncodeRenderPass();
	void encodeToSwapChain();

	void createDepthResources();
	void destroyDepthResources();

	vk::Format findDepthFormat();

	void createCommandPool(const PhysicalDeviceValues& values);

	void createPresentSyncObjects();
	void createRenderingFinishedSemaphores();

	void acquireNextSwapChainImage();

	bool recreateSwapChain();
	bool shouldRecreateSwapChain(SwapChainStatus status) const;

	void createImGuiDescriptorPool();
	void initImGui();
	void shutdownImGui();

	std::unique_ptr<os::GraphicsOperations> m_graphicsOps;

	vk::UniqueInstance m_vkInstance;
	vk::UniqueDebugReportCallbackEXT m_debugReport;            // legacy fallback (no VK_EXT_debug_utils)
	vk::UniqueDebugUtilsMessengerEXT m_debugMessenger;         // preferred debug callback

	vk::UniqueSurfaceKHR m_vkSurface;

	vk::UniqueDevice m_device;

	vk::Queue m_graphicsQueue;
	vk::Queue m_presentQueue;

	vk::UniqueSwapchainKHR m_swapChain;
	vk::Format m_swapChainImageFormat;
	vk::ColorSpaceKHR m_swapChainColorSpace = vk::ColorSpaceKHR::eSrgbNonlinear;
	bool m_hdrActive = false;            // True when an HDR10 (PQ/BT.2020) swap chain was negotiated
	bool m_hdrMetadataSupported = false; // VK_EXT_hdr_metadata device extension enabled
	vk::Extent2D m_swapChainExtent;
	vk::SurfaceTransformFlagBitsKHR m_swapChainPreTransform = vk::SurfaceTransformFlagBitsKHR::eIdentity;
	// The resolution the engine draws at (gr_screen.max_w/max_h), which is not
	// always the window size: when Cmdline_window_res is set,
	// SDLGraphicsOperations::createViewport sizes the window from that instead,
	// and the frame is scaled into it by the output-encode pass. -vr forces
	// exactly that split (Cmdline_window_res 1000x1000 vs. the VR resolution
	// option), so treating the two as one is what put whole frames in the
	// upper-left corner of the screen.
	vk::Extent2D m_renderExtent;
	SCP_vector<vk::Image> m_swapChainImages;
	SCP_vector<vk::UniqueImageView> m_swapChainImageViews;
	SCP_vector<vk::UniqueFramebuffer> m_swapChainFramebuffers;
	SCP_vector<VulkanRenderFrame*> m_swapChainImageRenderImage;
	// per image, not per frame: a frame's fence does not tell us when present has consumed the semaphore
	SCP_vector<vk::UniqueSemaphore> m_renderingFinishedSemaphores;

	// HDR composition pipeline: the whole frame is rendered into these fp16
	// images (via m_renderPass / m_swapChainFramebuffers) instead of directly
	// into the swap chain image. encodeToSwapChain() converts composition ->
	// swap chain: a direct blit (or encodeOutputPassthrough() as a fallback)
	// for SDR, or encodeOutput() (m_encodeRenderPass + m_encodeFramebuffers)
	// for the HDR10 PQ/BT.2020 transfer.
	SCP_vector<vk::UniqueImage> m_compositionImages;
	SCP_vector<vk::UniqueImageView> m_compositionImageViews;
	SCP_vector<VulkanAllocation> m_compositionAllocations;
	vk::UniqueSampler m_compositionSampler;
	SCP_vector<vk::UniqueFramebuffer> m_encodeFramebuffers;
	vk::UniqueRenderPass m_encodeRenderPass;

	uint32_t m_currentSwapChainImage = 0;
	uint32_t m_previousSwapChainImage = UINT32_MAX;  // For saveScreen() readback of previous frame

	// Depth buffer
	vk::UniqueImage m_depthImage;
	vk::UniqueImageView m_depthImageView;
	VulkanAllocation m_depthImageMemory;
	vk::Format m_depthFormat = vk::Format::eUndefined;

	vk::UniqueRenderPass m_renderPass;        // Swap chain pass with loadOp=eClear
	vk::UniqueRenderPass m_renderPassLoad;    // Swap chain pass with loadOp=eLoad (resumed after post-processing)
	vk::UniqueDescriptorPool m_imguiDescriptorPool;

	uint32_t m_currentFrame = 0;
	uint64_t m_frameNumber = 0;  // Total frames rendered (for sync tracking)
	std::array<std::unique_ptr<VulkanRenderFrame>, MAX_FRAMES_IN_FLIGHT> m_frames;

	vk::UniqueCommandPool m_graphicsCommandPool;

	// Current frame command buffer (valid between setupFrame and flip)
	vk::CommandBuffer m_currentCommandBuffer;
	SCP_vector<vk::CommandBuffer> m_currentCommandBuffers;  // For cleanup
	bool m_frameInProgress = false;
	bool m_pipelinesStale = false;

	// Swap chain recreation
	bool m_swapChainNeedsRecreation = false;

	// Physical device info (needed for memory manager)
	vk::PhysicalDevice m_physicalDevice;
	// Cached once at device selection: the limit/feature getters below are
	// hot-ish and querying the driver each call is wasteful. Populated alongside
	// m_physicalDevice in createLogicalDevice().
	vk::PhysicalDeviceProperties m_deviceProperties{};
	vk::PhysicalDeviceFeatures m_deviceFeatures{};
	uint32_t m_graphicsQueueFamilyIndex = 0;
	uint32_t m_presentQueueFamilyIndex = 0;

	// Memory, buffer, and texture management
	std::unique_ptr<VulkanMemoryManager> m_memoryManager;
	std::unique_ptr<VulkanBufferManager> m_bufferManager;
	std::unique_ptr<VulkanTextureManager> m_textureManager;
	std::unique_ptr<VulkanDeletionQueue> m_deletionQueue;

	// Shader, descriptor, and pipeline management
	std::unique_ptr<VulkanShaderManager> m_shaderManager;
	std::unique_ptr<VulkanDescriptorManager> m_descriptorManager;
	std::unique_ptr<VulkanPipelineManager> m_pipelineManager;

	// State tracking and draw management
	std::unique_ptr<VulkanStateTracker> m_stateTracker;
	std::unique_ptr<VulkanDrawManager> m_drawManager;

	// Query management (GPU timestamp profiling)
	std::unique_ptr<VulkanQueryManager> m_queryManager;

	// Raytraced shadow BLAS cache
	std::unique_ptr<VulkanRaytracingManager> m_raytracingManager;

	// Post-processing
	std::unique_ptr<VulkanPostProcessor> m_postProcessor;
	bool m_sceneRendering = false;
	bool m_sceneDepthCopiedThisFrame = false;
	bool m_sceneDepthSaved = false;    // True between saveSceneDepth() and restoreSceneDepth()
	bool m_useGbufRenderPass = false;  // True when scene uses G-buffer (deferred lighting)

	bool m_supportsShaderViewportLayerOutput = false;  // VK_EXT_shader_viewport_index_layer
	bool m_supportsRaytracedShadows = false;  // VK_KHR_ray_query + VK_KHR_acceleration_structure + VK_KHR_deferred_host_operations
	vk::SampleCountFlagBits m_msaaSampleCount = vk::SampleCountFlagBits::e1;  // Validated MSAA sample count
	bool m_renderTargetActive = false;  // True when rendering to off-screen RT (bm_set_render_target)

	bool m_debugReportEnabled = false;
	bool m_debugUtilsEnabled = false;

};

} // namespace graphics::vulkan