#include "VulkanOpenXR.h" #include "VulkanBarrier.h" #include "VulkanRenderer.h" #include "gr_vulkan.h" #include "graphics/2d.h" #include "graphics/openxr.h" #ifdef FS_OPENXR #include "io/cursor.h" #include "io/mouse.h" #include #define XR_USE_GRAPHICS_API_VULKAN #include "graphics/openxr_internal.h" #endif namespace graphics::vulkan { namespace { #ifdef FS_OPENXR // Set while creating the Vulkan instance (the only point at which the runtime's // graphics requirements can be checked, since they must be queried *before* // instance creation) and read back later by test_capabilities(). bool Xr_graphics_requirements_met = false; std::array, 2> Xr_swapchain_images; // Which eyes hold an acquired swapchain image for the frame currently being // built. A stereo frame spans two gr_flip() calls, so the left eye's // acquisition has to survive until the right eye's call submits and releases // both -- the release cannot happen before the work is submitted. std::array Xr_eye_acquired = {false, false}; /** * @brief Copy the frame the engine just rendered into one eye's swapchain image * * The source is the renderer's composition image, which is the Vulkan * equivalent of the backbuffer the OpenGL path blits from: the whole frame -- * 3D scene, HUD, menus -- has landed there by the time gr_flip() runs. Note * that it is sampled before the output-encode pass, so the user's gamma slider * (Gr_gamma) applies to the desktop mirror but not to the HMD image. * * Must be called with no render pass active on the frame command buffer. */ void blit_to_swapchain_image(VulkanRenderer* renderer, vk::Image dst, uint32_t dstWidth, uint32_t dstHeight) { const vk::CommandBuffer cmd = renderer->getCurrentCommandBuffer(); const vk::Image composition = renderer->getCurrentCompositionImage(); const vk::Extent2D srcExtent = renderer->getRenderExtent(); if (!cmd || !composition || !dst) { return; } // The composition image is left in eShaderReadOnlyOptimal by the render // pass that just ended, and must be put back that way afterwards: it is // what both m_renderPassLoad's initialLayout and the output-encode pass // expect to find. std::array preBarriers; preBarriers[0].image = composition; preBarriers[0].oldLayout = vk::ImageLayout::eShaderReadOnlyOptimal; preBarriers[0].newLayout = vk::ImageLayout::eTransferSrcOptimal; preBarriers[0].srcStage = vk::PipelineStageFlagBits2::eColorAttachmentOutput; preBarriers[0].srcAccess = vk::AccessFlagBits2::eColorAttachmentWrite; preBarriers[0].dstStage = vk::PipelineStageFlagBits2::eBlit; preBarriers[0].dstAccess = vk::AccessFlagBits2::eTransferRead; // eUndefined discards whatever the runtime last left in the image, which is // exactly right: the blit overwrites every pixel of it. preBarriers[1].image = dst; preBarriers[1].oldLayout = vk::ImageLayout::eUndefined; preBarriers[1].newLayout = vk::ImageLayout::eTransferDstOptimal; preBarriers[1].srcStage = vk::PipelineStageFlagBits2::eNone; preBarriers[1].srcAccess = {}; preBarriers[1].dstStage = vk::PipelineStageFlagBits2::eBlit; preBarriers[1].dstAccess = vk::AccessFlagBits2::eTransferWrite; cmdImageBarriers(cmd, ArrayView(preBarriers.data(), preBarriers.size())); vk::ImageBlit blit; blit.srcSubresource.aspectMask = vk::ImageAspectFlagBits::eColor; blit.srcSubresource.mipLevel = 0; blit.srcSubresource.baseArrayLayer = 0; blit.srcSubresource.layerCount = 1; blit.srcOffsets[0] = vk::Offset3D(0, 0, 0); blit.srcOffsets[1] = vk::Offset3D(static_cast(srcExtent.width), static_cast(srcExtent.height), 1); blit.dstSubresource.aspectMask = vk::ImageAspectFlagBits::eColor; blit.dstSubresource.mipLevel = 0; blit.dstSubresource.baseArrayLayer = 0; blit.dstSubresource.layerCount = 1; blit.dstOffsets[0] = vk::Offset3D(0, 0, 0); blit.dstOffsets[1] = vk::Offset3D(static_cast(dstWidth), static_cast(dstHeight), 1); // eLinear because the eye resolution rarely matches the window resolution, // matching the OpenGL path's GL_LINEAR blit into the swapchain. cmd.blitImage(composition, vk::ImageLayout::eTransferSrcOptimal, dst, vk::ImageLayout::eTransferDstOptimal, 1, &blit, vk::Filter::eLinear); std::array postBarriers; // Runtimes expect a released swapchain image in eColorAttachmentOptimal. postBarriers[0].image = dst; postBarriers[0].oldLayout = vk::ImageLayout::eTransferDstOptimal; postBarriers[0].newLayout = vk::ImageLayout::eColorAttachmentOptimal; postBarriers[0].srcStage = vk::PipelineStageFlagBits2::eBlit; postBarriers[0].srcAccess = vk::AccessFlagBits2::eTransferWrite; postBarriers[0].dstStage = vk::PipelineStageFlagBits2::eColorAttachmentOutput; postBarriers[0].dstAccess = vk::AccessFlagBits2::eColorAttachmentWrite; postBarriers[1].image = composition; postBarriers[1].oldLayout = vk::ImageLayout::eTransferSrcOptimal; postBarriers[1].newLayout = vk::ImageLayout::eShaderReadOnlyOptimal; postBarriers[1].srcStage = vk::PipelineStageFlagBits2::eBlit; postBarriers[1].srcAccess = vk::AccessFlagBits2::eTransferRead; postBarriers[1].dstStage = vk::PipelineStageFlagBits2::eFragmentShader; postBarriers[1].dstAccess = vk::AccessFlagBits2::eShaderSampledRead; cmdImageBarriers(cmd, ArrayView(postBarriers.data(), postBarriers.size())); } /** * @brief Acquire one eye's swapchain image and blit the current frame into it * * Ends the in-flight render pass, since a blit cannot be recorded inside one. */ bool render_eye(VulkanRenderer* renderer, uint32_t eye) { renderer->endCurrentRenderPass(); XrSwapchain swapchain = xr_swapchains[eye]->swapchain; XrSwapchainImageAcquireInfo acquireImageInfo{ XR_TYPE_SWAPCHAIN_IMAGE_ACQUIRE_INFO, nullptr }; uint32_t activeIndex = 0; if (xrAcquireSwapchainImage(swapchain, &acquireImageInfo, &activeIndex) != XR_SUCCESS) { return false; } XrSwapchainImageWaitInfo waitImageInfo{ XR_TYPE_SWAPCHAIN_IMAGE_WAIT_INFO, nullptr, std::numeric_limits::max() }; xrWaitSwapchainImage(swapchain, &waitImageInfo); Xr_eye_acquired[eye] = true; blit_to_swapchain_image(renderer, vk::Image(Xr_swapchain_images[eye][activeIndex].image), xr_swapchains[eye]->width, xr_swapchains[eye]->height); return true; } void release_acquired_images() { for (uint32_t i = 0; i < 2; i++) { if (!Xr_eye_acquired[i]) { continue; } XrSwapchainImageReleaseInfo releaseImageInfo{ XR_TYPE_SWAPCHAIN_IMAGE_RELEASE_INFO, nullptr }; xrReleaseSwapchainImage(xr_swapchains[i]->swapchain, &releaseImageInfo); Xr_eye_acquired[i] = false; } } /** * @brief Draw the mouse cursor into the frame * * The cursor is an OS cursor, so it is drawn by the compositor over the window * and never appears in anything we render. On a flat screen inside the headset * that would leave the player clicking blind, so it has to be stamped into the * image by hand -- as the OpenGL path also does. Must run while a render pass * is still active. */ void draw_cursor() { auto* cursorManager = io::mouse::CursorManager::get(); if (cursorManager == nullptr || !cursorManager->isCursorShown()) { return; } auto* cursor = cursorManager->getCurrentCursor(); if (cursor == nullptr) { return; } const int cursorBitmap = cursor->getBitmapHandle(); if (cursorBitmap < 0) { return; } int mouseX, mouseY; mouse_get_pos(&mouseX, &mouseY); gr_set_bitmap(cursorBitmap); gr_bitmap(mouseX, mouseY, GR_RESIZE_NONE); } /** * @brief Submit the composed frame to both the HMD and the desktop mirror */ void end_frame(VulkanRenderer* renderer, bool stereo) { // Submit first: OpenXR requires every command that writes a swapchain image // to have been submitted to the queue the session was created with before // xrEndFrame is called. This also presents the desktop mirror and acquires // the next swap chain image. renderer->flip(); release_acquired_images(); XrCompositionLayerProjectionView projectedViews[2]; XrCompositionLayerProjection projectionLayer{}; XrCompositionLayerQuad quadLayer{}; const XrCompositionLayerBaseHeader* pLayer = nullptr; if (stereo) { for (uint32_t i = 0; i < 2; i++) { projectedViews[i].type = XR_TYPE_COMPOSITION_LAYER_PROJECTION_VIEW; projectedViews[i].next = nullptr; projectedViews[i].pose = xr_views[i].pose; projectedViews[i].fov = xr_views[i].fov; projectedViews[i].subImage = { xr_swapchains[i]->swapchain, { { 0, 0 }, { (int32_t)xr_swapchains[i]->width, (int32_t)xr_swapchains[i]->height } }, 0 }; } projectionLayer.type = XR_TYPE_COMPOSITION_LAYER_PROJECTION; projectionLayer.next = nullptr; projectionLayer.layerFlags = 0; projectionLayer.space = xr_space; projectionLayer.viewCount = 2; projectionLayer.views = projectedViews; pLayer = (const XrCompositionLayerBaseHeader*)&projectionLayer; } else { // A flat frame (menus, briefings, anything not rendered per-eye) becomes // a quad layer rather than a projection layer, letting the runtime place // and filter a single 2D image instead of us reprojecting it per eye. // Position and size reproduce the proportions of the OpenGL path's quad: // 4 units ahead of the origin, 4 units tall, widened by the aspect ratio. // OpenXR's forward axis is -Z, FSO's is +Z. quadLayer.type = XR_TYPE_COMPOSITION_LAYER_QUAD; quadLayer.next = nullptr; quadLayer.layerFlags = 0; quadLayer.space = xr_space; quadLayer.eyeVisibility = XR_EYE_VISIBILITY_BOTH; quadLayer.subImage = { xr_swapchains[0]->swapchain, { { 0, 0 }, { (int32_t)xr_swapchains[0]->width, (int32_t)xr_swapchains[0]->height } }, 0 }; quadLayer.pose = XrPosef{ XrQuaternionf{ 0, 0, 0, 1 }, XrVector3f{ 0, 0, -4.0f } }; quadLayer.size = XrExtent2Df{ 4.0f * gr_screen.clip_aspect, 4.0f }; pLayer = (const XrCompositionLayerBaseHeader*)&quadLayer; } XrFrameEndInfo frameEndInfo{ XR_TYPE_FRAME_END_INFO, nullptr, xr_state.predictedDisplayTime, XR_ENVIRONMENT_BLEND_MODE_OPAQUE, 0, nullptr }; if (xr_state.shouldRender) { frameEndInfo.layerCount = 1; frameEndInfo.layers = &pLayer; } xrEndFrame(xr_session, &frameEndInfo); } #endif } // namespace // SETUP FUNCTIONS VULKAN #ifdef FS_OPENXR SCP_vector vulkan_openxr_get_extensions() { return { XR_KHR_VULKAN_ENABLE2_EXTENSION_NAME }; } bool vulkan_openxr_test_capabilities() { // The real check happened back when the Vulkan instance was created -- the // runtime requires its graphics requirements to be queried before that // point, which is far earlier than this function is called. return Xr_graphics_requirements_met; } bool vulkan_openxr_create_session() { auto* renderer = getRendererInstance(); XrGraphicsBindingVulkan2KHR graphicsBinding { XR_TYPE_GRAPHICS_BINDING_VULKAN2_KHR, nullptr, renderer->getVkInstance(), renderer->getPhysicalDevice(), renderer->getDevice(), renderer->getGraphicsQueueFamilyIndex(), 0 }; XrSessionCreateInfo sessionCreateInfo { XR_TYPE_SESSION_CREATE_INFO, &graphicsBinding, 0, xr_system }; XrResult sessionInit = xrCreateSession(xr_instance, &sessionCreateInfo, &xr_session); if (sessionInit != XR_SUCCESS) { mprintf(("Failed to create OpenXR session with code %d\n", static_cast(sessionInit))); return false; } return true; } int64_t vulkan_openxr_get_swapchain_format(const SCP_vector& allowed) { // Prefer the format of our own composition image, which makes the blit into // the swapchain a straight copy with no conversion. const int64_t preferred[] = { static_cast(VK_FORMAT_R16G16B16A16_SFLOAT), static_cast(VK_FORMAT_B8G8R8A8_UNORM), }; for (int64_t format : preferred) { if (std::find(allowed.cbegin(), allowed.cend(), format) != allowed.cend()) { return format; } } return allowed.front(); } bool vulkan_openxr_acquire_swapchain_buffers() { for (uint32_t i = 0; i < 2; i++) { uint32_t imageCount = 0; XrResult swapchainAcq = xrEnumerateSwapchainImages(xr_swapchains[i]->swapchain, 0, &imageCount, nullptr); if (swapchainAcq != XR_SUCCESS) { mprintf(("Failed to acquire OpenXR swapchain %d with code %d\n", i, static_cast(swapchainAcq))); return false; } Xr_swapchain_images[i] = SCP_vector(imageCount, { XR_TYPE_SWAPCHAIN_IMAGE_VULKAN2_KHR, nullptr, VK_NULL_HANDLE }); swapchainAcq = xrEnumerateSwapchainImages(xr_swapchains[i]->swapchain, imageCount, &imageCount, (XrSwapchainImageBaseHeader*)Xr_swapchain_images[i].data()); if (swapchainAcq != XR_SUCCESS) { mprintf(("Failed to acquire OpenXR swapchain %d with code %d\n", i, static_cast(swapchainAcq))); return false; } } return true; } bool vulkan_openxr_flip() { if (!openxr_enabled()) { return false; } auto* renderer = getRendererInstance(); static bool was_multiframe = false; switch (xr_stage) { case OpenXRFBStage::FIRST: // openxr_start_frame() already ran in openxr_start_stereo_frame(). was_multiframe = true; render_eye(renderer, 0); // The right eye is rendered into the same composition image over the // next gr_flip() cycle, so start it from a clean slate rather than on // top of the left eye's picture. renderer->restartCompositionPass(); xr_stage = OpenXRFBStage::SECOND; return true; case OpenXRFBStage::SECOND: render_eye(renderer, 1); end_frame(renderer, true); break; case OpenXRFBStage::NONE: // A flat frame that nothing has told OpenXR about yet. openxr_start_frame(); if (was_multiframe) { openxr_reset_offset(); was_multiframe = false; } draw_cursor(); render_eye(renderer, 0); end_frame(renderer, false); break; } xr_stage = OpenXRFBStage::NONE; // gr_flip() skips its own page flip when we return true, so the new frame // has to be opened here instead. gr_setup_frame(); return true; } #else // Stubs for builds without OpenXR support. SCP_vector vulkan_openxr_get_extensions() { return SCP_vector{}; } bool vulkan_openxr_test_capabilities() { return false; } bool vulkan_openxr_create_session() { return false; } int64_t vulkan_openxr_get_swapchain_format(const SCP_vector& /*allowed*/) { return 0; } bool vulkan_openxr_acquire_swapchain_buffers() { return false; } bool vulkan_openxr_flip() { return false; } #endif vk::UniqueInstance vulkan_openxr_create_instance(const vk::InstanceCreateInfo& createInfo) { #ifdef FS_OPENXR // Must be queried before the instance is created, per the extension. XrGraphicsRequirementsVulkan2KHR requirements{ XR_TYPE_GRAPHICS_REQUIREMENTS_VULKAN2_KHR, nullptr, 0, 0 }; auto reqResult = openxr_callExtensionFunction( "xrGetVulkanGraphicsRequirements2KHR", xr_instance, xr_system, &requirements); if (!reqResult.has_value() || *reqResult != XR_SUCCESS) { mprintf(("Failed to query OpenXR graphics requirements!\n")); return vk::createInstanceUnique(createInfo, nullptr); } const XrVersion ourVersion = XR_MAKE_VERSION(VK_API_VERSION_MAJOR(VulkanApiVersion), VK_API_VERSION_MINOR(VulkanApiVersion), 0); if (ourVersion < requirements.minApiVersionSupported || ourVersion > requirements.maxApiVersionSupported) { mprintf(("System doesn't meet OpenXR graphics requirements (min %" PRIu64 ", max %" PRIu64 ", using %" PRIu64 ")!\n", requirements.minApiVersionSupported, requirements.maxApiVersionSupported, ourVersion)); return vk::createInstanceUnique(createInfo, nullptr); } XrVulkanInstanceCreateInfoKHR xrCreateInfo{ XR_TYPE_VULKAN_INSTANCE_CREATE_INFO_KHR, nullptr, xr_system, 0, reinterpret_cast(SDL_Vulkan_GetVkGetInstanceProcAddr()), reinterpret_cast(&createInfo), nullptr }; VkInstance rawInstance = VK_NULL_HANDLE; VkResult vulkanResult = VK_SUCCESS; auto created = openxr_callExtensionFunction( "xrCreateVulkanInstanceKHR", xr_instance, &xrCreateInfo, &rawInstance, &vulkanResult); if (!created.has_value() || *created != XR_SUCCESS || vulkanResult != VK_SUCCESS) { mprintf(("OpenXR failed to create the Vulkan instance (xr %d, vk %d)!\n", created.has_value() ? static_cast(*created) : -1, static_cast(vulkanResult))); return vk::createInstanceUnique(createInfo, nullptr); } Xr_graphics_requirements_met = true; const vk::detail::ObjectDestroy deleter(nullptr, VULKAN_HPP_DEFAULT_DISPATCHER); return vk::UniqueInstance(vk::Instance(rawInstance), deleter); #else return vk::createInstanceUnique(createInfo, nullptr); #endif } vk::PhysicalDevice vulkan_openxr_required_physical_device([[maybe_unused]] vk::Instance instance) { #ifdef FS_OPENXR if (openxr_requested() && Xr_graphics_requirements_met) { XrVulkanGraphicsDeviceGetInfoKHR getInfo{ XR_TYPE_VULKAN_GRAPHICS_DEVICE_GET_INFO_KHR, nullptr, xr_system, instance }; VkPhysicalDevice rawDevice = VK_NULL_HANDLE; auto result = openxr_callExtensionFunction( "xrGetVulkanGraphicsDevice2KHR", xr_instance, &getInfo, &rawDevice); if (result.has_value() && *result == XR_SUCCESS && rawDevice != VK_NULL_HANDLE) { vk::PhysicalDevice device(rawDevice); mprintf(("OpenXR requires Vulkan device %s.\n", device.getProperties().deviceName.data())); return device; } mprintf(("Failed to query the Vulkan device required by OpenXR!\n")); } #endif return {}; } vk::UniqueDevice vulkan_openxr_create_device(vk::PhysicalDevice physicalDevice, const vk::DeviceCreateInfo& createInfo) { #ifdef FS_OPENXR if (Xr_graphics_requirements_met) { XrVulkanDeviceCreateInfoKHR xrCreateInfo{ XR_TYPE_VULKAN_DEVICE_CREATE_INFO_KHR, nullptr, xr_system, 0, reinterpret_cast(SDL_Vulkan_GetVkGetInstanceProcAddr()), physicalDevice, reinterpret_cast(&createInfo), nullptr }; VkDevice rawDevice = VK_NULL_HANDLE; VkResult vulkanResult = VK_SUCCESS; auto created = openxr_callExtensionFunction( "xrCreateVulkanDeviceKHR", xr_instance, &xrCreateInfo, &rawDevice, &vulkanResult); if (created.has_value() && *created == XR_SUCCESS && vulkanResult == VK_SUCCESS) { const vk::detail::ObjectDestroy deleter(nullptr, VULKAN_HPP_DEFAULT_DISPATCHER); return vk::UniqueDevice(vk::Device(rawDevice), deleter); } // Falling back to a device the runtime did not create means no VR, but // it is better than failing to start the game at all. mprintf(("OpenXR failed to create the Vulkan device (xr %d, vk %d)!\n", created.has_value() ? static_cast(*created) : -1, static_cast(vulkanResult))); Xr_graphics_requirements_met = false; } #endif return physicalDevice.createDeviceUnique(createInfo); } } // namespace graphics::vulkan