Library Forwarding: Support Vulkan forwarding with guest-libX11

Data/ThunksDB.json

@@ -30,16 +30,10 @@
     },
     "Vulkan": {
       "Library": "libvulkan-guest.so",
-      "Depends": [
-        "xcb"
-      ],
       "Overlay": [
         "@PREFIX_LIB@/libvulkan.so",
         "@PREFIX_LIB@/libvulkan.so.1",
         "@HOME@/.local/share/Steam/ubuntu12_32/steam-runtime/pinned_libs_64/libvulkan.so.1"
-      ],
-      "Comment": [
-        "Vulkan library relies on xcb, otherwise it crashes with jemalloc"
       ]
     },
     "xcb": {

ThunkLibs/Generator/analysis.cpp

@@ -545,6 +545,10 @@ void AnalysisAction::CoverReferencedTypes(clang::ASTContext& context) {
 
       for (auto* member : type->getAsStructureType()->getDecl()->fields()) {
         auto member_type = member->getType().getTypePtr();
+        if (type_repack_info.UsesCustomRepackFor(member) && member_type->isPointerType() && member_type->getPointeeType()->isStructureType()) {
+          continue;
+        }
+
         while (member_type->isArrayType()) {
           member_type = member_type->getArrayElementTypeNoTypeQual();
         }

ThunkLibs/libvulkan/Guest.cpp

@@ -84,4 +84,12 @@ PFN_vkVoidFunction vkGetInstanceProcAddr(VkInstance a_0, const char* a_1) {
 }
 }
 
-LOAD_LIB(libvulkan)
+void OnInit() {
+  // TODO: Load libX11 on-demand instead
+  void* libx11 = dlopen("libX11.so.6", RTLD_LAZY);
+  fexfn_pack_SetGuestXSync((uintptr_t)dlsym(libx11, "XSync"), (uintptr_t)CallbackUnpack<decltype(XSync)>::Unpack);
+  fexfn_pack_SetGuestXGetVisualInfo((uintptr_t)dlsym(libx11, "XGetVisualInfo"), (uintptr_t)CallbackUnpack<decltype(XGetVisualInfo)>::Unpack);
+  fexfn_pack_SetGuestXDisplayString((uintptr_t)dlsym(libx11, "XDisplayString"), (uintptr_t)CallbackUnpack<decltype(XDisplayString)>::Unpack);
+}
+
+LOAD_LIB_INIT(libvulkan, OnInit)

ThunkLibs/libvulkan/Host.cpp

@@ -16,13 +16,12 @@ $end_info$
 
 #include <cstring>
 #include <mutex>
-#include <unordered_map>
 #include <string_view>
 
-#include <dlfcn.h>
-
 #include "thunkgen_host_libvulkan.inl"
 
+#include <common/X11Manager.h>
+
 static bool SetupInstance {};
 static std::mutex SetupMutex {};
 
@@ -50,6 +49,67 @@ static void DoSetupWithInstance(VkInstance instance) {
 
 #define FEXFN_IMPL(fn) fexfn_impl_libvulkan_##fn
 
+static X11Manager x11_manager;
+
+static void fexfn_impl_libvulkan_SetGuestXGetVisualInfo(uintptr_t GuestTarget, uintptr_t GuestUnpacker) {
+  MakeHostTrampolineForGuestFunctionAt(GuestTarget, GuestUnpacker, &x11_manager.GuestXGetVisualInfo);
+}
+
+static void fexfn_impl_libvulkan_SetGuestXSync(uintptr_t GuestTarget, uintptr_t GuestUnpacker) {
+  MakeHostTrampolineForGuestFunctionAt(GuestTarget, GuestUnpacker, &x11_manager.GuestXSync);
+}
+
+static void fexfn_impl_libvulkan_SetGuestXDisplayString(uintptr_t GuestTarget, uintptr_t GuestUnpacker) {
+  MakeHostTrampolineForGuestFunctionAt(GuestTarget, GuestUnpacker, &x11_manager.GuestXDisplayString);
+}
+
+void fex_custom_repack_entry(host_layout<VkXcbSurfaceCreateInfoKHR>& to, const guest_layout<VkXcbSurfaceCreateInfoKHR>& from) {
+  // TODO: xcb_aux_sync?
+  to.data.connection = x11_manager.GuestToHostConnection(const_cast<xcb_connection_t*>(from.data.connection.force_get_host_pointer()));
+}
+
+bool fex_custom_repack_exit(guest_layout<VkXcbSurfaceCreateInfoKHR>&, const host_layout<VkXcbSurfaceCreateInfoKHR>&) {
+  // TODO: xcb_sync?
+  return false;
+}
+
+void fex_custom_repack_entry(host_layout<VkXlibSurfaceCreateInfoKHR>& to, const guest_layout<VkXlibSurfaceCreateInfoKHR>& from) {
+  to.data.dpy = x11_manager.GuestToHostDisplay(const_cast<Display*>(from.data.dpy.force_get_host_pointer()));
+}
+
+bool fex_custom_repack_exit(guest_layout<VkXlibSurfaceCreateInfoKHR>&, const host_layout<VkXlibSurfaceCreateInfoKHR>& from) {
+  x11_manager.HostXFlush(from.data.dpy);
+  return false;
+}
+
+static VkResult fexfn_impl_libvulkan_vkAcquireXlibDisplayEXT(VkPhysicalDevice a_0, guest_layout<Display*> a_1, VkDisplayKHR a_2) {
+  auto host_display = x11_manager.GuestToHostDisplay(a_1.force_get_host_pointer());
+  auto ret = fexldr_ptr_libvulkan_vkAcquireXlibDisplayEXT(a_0, host_display, a_2);
+  x11_manager.HostXFlush(host_display);
+  return ret;
+}
+
+static VkResult fexfn_impl_libvulkan_vkGetRandROutputDisplayEXT(VkPhysicalDevice a_0, guest_layout<Display*> a_1, RROutput a_2, VkDisplayKHR* a_3) {
+  auto host_display = x11_manager.GuestToHostDisplay(a_1.force_get_host_pointer());
+  auto ret = fexldr_ptr_libvulkan_vkGetRandROutputDisplayEXT(a_0, host_display, a_2, a_3);
+  x11_manager.HostXFlush(host_display);
+  return ret;
+}
+
+static VkBool32 fexfn_impl_libvulkan_vkGetPhysicalDeviceXcbPresentationSupportKHR(VkPhysicalDevice a_0, uint32_t a_1,
+                                                                                  guest_layout<xcb_connection_t*> a_2, xcb_visualid_t a_3) {
+  auto host_connection = x11_manager.GuestToHostConnection(a_2.force_get_host_pointer());
+  return fexldr_ptr_libvulkan_vkGetPhysicalDeviceXcbPresentationSupportKHR(a_0, a_1, host_connection, a_3);
+}
+
+static VkBool32 fexfn_impl_libvulkan_vkGetPhysicalDeviceXlibPresentationSupportKHR(VkPhysicalDevice a_0, uint32_t a_1,
+                                                                                   guest_layout<Display*> a_2, VisualID a_3) {
+  auto host_display = x11_manager.GuestToHostDisplay(a_2.force_get_host_pointer());
+  auto ret = fexldr_ptr_libvulkan_vkGetPhysicalDeviceXlibPresentationSupportKHR(a_0, a_1, host_display, a_3);
+  x11_manager.HostXFlush(host_display);
+  return ret;
+}
+
 // Functions with callbacks are overridden to ignore the guest-side callbacks
 
 static VkResult
@@ -146,6 +206,14 @@ static PFN_vkVoidFunction LookupCustomVulkanFunction(const char* a_1) {
     return (PFN_vkVoidFunction)fexfn_impl_libvulkan_vkAllocateMemory;
   } else if (a_1 == "vkFreeMemory"sv) {
     return (PFN_vkVoidFunction)fexfn_impl_libvulkan_vkFreeMemory;
+  } else if (a_1 == "vkAcquireXlibDisplayEXT"sv) {
+    return (PFN_vkVoidFunction)fexfn_impl_libvulkan_vkAcquireXlibDisplayEXT;
+  } else if (a_1 == "vkGetRandROutputDisplayEXT"sv) {
+    return (PFN_vkVoidFunction)fexfn_impl_libvulkan_vkGetRandROutputDisplayEXT;
+  } else if (a_1 == "vkGetPhysicalDeviceXcbPresentationSupportKHR"sv) {
+    return (PFN_vkVoidFunction)fexfn_impl_libvulkan_vkGetPhysicalDeviceXcbPresentationSupportKHR;
+  } else if (a_1 == "vkGetPhysicalDeviceXlibPresentationSupportKHR"sv) {
+    return (PFN_vkVoidFunction)fexfn_impl_libvulkan_vkGetPhysicalDeviceXlibPresentationSupportKHR;
   }
   return nullptr;
 }

ThunkLibs/libvulkan/libvulkan_interface.cpp

@@ -28,6 +28,17 @@ struct fex_gen_config<vkGetInstanceProcAddr> : fexgen::custom_host_impl, fexgen:
 template<typename>
 struct fex_gen_type {};
 
+// internal use
+void SetGuestXSync(uintptr_t, uintptr_t);
+void SetGuestXGetVisualInfo(uintptr_t, uintptr_t);
+void SetGuestXDisplayString(uintptr_t, uintptr_t);
+template<>
+struct fex_gen_config<SetGuestXSync> : fexgen::custom_guest_entrypoint, fexgen::custom_host_impl {};
+template<>
+struct fex_gen_config<SetGuestXGetVisualInfo> : fexgen::custom_guest_entrypoint, fexgen::custom_host_impl {};
+template<>
+struct fex_gen_config<SetGuestXDisplayString> : fexgen::custom_guest_entrypoint, fexgen::custom_host_impl {};
+
 // So-called "dispatchable" handles are represented as opaque pointers.
 // In addition to marking them as such, API functions that create these objects
 // need special care since they wrap these handles in another pointer, which
@@ -91,9 +102,9 @@ struct fex_gen_type<VkAllocationCallbacks> : fexgen::opaque_type {};
 
 // X11 interop
 template<>
-struct fex_gen_type<Display> : fexgen::opaque_type {};
+struct fex_gen_config<&VkXcbSurfaceCreateInfoKHR::connection> : fexgen::custom_repack {};
 template<>
-struct fex_gen_type<xcb_connection_t> : fexgen::opaque_type {};
+struct fex_gen_config<&VkXlibSurfaceCreateInfoKHR::dpy> : fexgen::custom_repack {};
 
 // Wayland interop
 template<>
@@ -1398,9 +1409,13 @@ struct fex_gen_config<vkCmdDrawMeshTasksIndirectCountEXT> {};
 
 // vulkan_xlib_xrandr.h
 template<>
-struct fex_gen_config<vkAcquireXlibDisplayEXT> {};
+struct fex_gen_config<vkAcquireXlibDisplayEXT> : fexgen::custom_host_impl {};
 template<>
-struct fex_gen_config<vkGetRandROutputDisplayEXT> {};
+struct fex_gen_param<vkAcquireXlibDisplayEXT, 1, Display*> : fexgen::ptr_passthrough {};
+template<>
+struct fex_gen_config<vkGetRandROutputDisplayEXT> : fexgen::custom_host_impl {};
+template<>
+struct fex_gen_param<vkGetRandROutputDisplayEXT, 1, Display*> : fexgen::ptr_passthrough {};
 
 // vulkan_wayland.h
 template<>
@@ -1412,11 +1427,16 @@ struct fex_gen_config<vkGetPhysicalDeviceWaylandPresentationSupportKHR> {};
 template<>
 struct fex_gen_config<vkCreateXcbSurfaceKHR> {};
 template<>
-struct fex_gen_config<vkGetPhysicalDeviceXcbPresentationSupportKHR> {};
+struct fex_gen_config<vkGetPhysicalDeviceXcbPresentationSupportKHR> : fexgen::custom_host_impl {};
+template<>
+struct fex_gen_param<vkGetPhysicalDeviceXcbPresentationSupportKHR, 2, xcb_connection_t*> : fexgen::ptr_passthrough {};
 
 // vulkan_xlib.h
 template<>
 struct fex_gen_config<vkCreateXlibSurfaceKHR> {};
 template<>
-struct fex_gen_config<vkGetPhysicalDeviceXlibPresentationSupportKHR> {};
+struct fex_gen_config<vkGetPhysicalDeviceXlibPresentationSupportKHR> : fexgen::custom_host_impl {};
+template<>
+struct fex_gen_param<vkGetPhysicalDeviceXlibPresentationSupportKHR, 2, Display*> : fexgen::ptr_passthrough {};
+
 } // namespace internal

unittests/ThunkLibs/abi.cpp

@@ -615,7 +615,8 @@ TEST_CASE_METHOD(Fixture, "DataLayoutPointers") {
                                         "struct B { C* a; int16_t b; };\n"
                                         "struct A { int32_t a; B b; };\n"
                                         "template<> struct fex_gen_config<&B::a> : fexgen::custom_repack {};\n"
-                                        "template<> struct fex_gen_type<A> {};\n",
+                                        "template<> struct fex_gen_type<A> {};\n"
+                                        "template<> struct fex_gen_type<C> {};\n",
                                         guest_abi);
 
       INFO(FormatDataLayout(action->host_layout));
@@ -699,7 +700,8 @@ TEST_CASE_METHOD(Fixture, "DataLayoutPointers") {
                                       "#endif\n"
                                       "struct A { B* a; };\n"
                                       "template<> struct fex_gen_config<&A::a> : fexgen::custom_repack {};\n"
-                                      "template<> struct fex_gen_type<A> {};\n",
+                                      "template<> struct fex_gen_type<A> {};\n"
+                                      "template<> struct fex_gen_type<B> {};\n",
                                       guest_abi);
 
     INFO(FormatDataLayout(action->host_layout));
@@ -720,15 +722,16 @@ TEST_CASE_METHOD(Fixture, "DataLayoutPointers") {
                                       "struct B {};\n"
                                       "struct A { B* a; };\n"
                                       "template<> struct fex_gen_config<&A::a> : fexgen::custom_repack {};\n"
-                                      "template<> struct fex_gen_type<A> {};\n",
+                                      "template<> struct fex_gen_type<A> {};\n"
+                                      "template<> struct fex_gen_type<B> {};\n",
                                       guest_abi);
 
     INFO(FormatDataLayout(action->host_layout));
 
     REQUIRE(action->guest_layout->contains("A"));
     REQUIRE(action->guest_layout->contains("B"));
-    CHECK(action->GetTypeCompatibility("struct B") == TypeCompatibility::Full);
     CHECK(action->GetTypeCompatibility("struct A") == TypeCompatibility::Repackable);
+    CHECK(action->GetTypeCompatibility("struct B") == TypeCompatibility::Full);
   }
 
   SECTION("Self-referencing struct (like VkBaseOutStructure)") {