gamescope: EA sign-in stays above the EA launcher's splash (#2037)

* gamescope: EA sign-in stays above the EA launcher's splash

The EA app's Steam launcher shows the game's splash as a layered tool
window, and the EA app, another process, opens its sign-in window
topmost. gamescope took the splash as the sign-in's override window, so
the splash was painted over the sign-in and became gamescope's keyboard
focus window (Burnout Paradise Remastered). With patch 0126 a window from
another process that is neither topmost nor transient for a topmost
focus window is no longer an override candidate for it, as on Windows.
map_win now also reads _WINE_HWND_EXSTYLE: gamescope only took it from a
PropertyNotify, which it listens for from map on, so a style Wine set
before mapping was never known.

* gamescope: the topmost rule needs both windows' pids

gamescope keeps a window's pid at -1 when XRes cannot name its client,
so an unknown pid next to a known one is not evidence of two processes.
Patch 0126 now applies the rule only when both pids are known. The patch
carries function names in its hunk headers, and the test also checks
that its map_win hunk reads _WINE_HWND_EXSTYLE into hasHwndStyleEx (a
check of the patch text, not of a running gamescope).

tools/gamescope/PATCHES.md

@@ -209,3 +209,19 @@ the connector's windows (`mangoapp_nudge_app_frame`). With `DROIDDECK_HUD_FRAMES
 sets `/run/droiddeck-hud/frames`), gamescope maps that file and writes the same commits of connector 0,
 whether or not a mangoapp runs, plus the focused pid, output size, refresh, Steam focus and engine name.
 The layout is in the comment above `droiddeck_hud_frames_map`; the app reads it in `hudframes.c`.
+
+## 0126 steamcompmgr: a topmost focus window keeps other processes' windows beneath it
+
+EA games on Steam start through the EA app. Its Steam launcher (`EASteamLauncher.exe`) shows the
+game's splash as a layered tool window, and the EA app (`EADesktop.exe`, another process) opens its
+sign-in window `WS_EX_TOPMOST`. gamescope took the splash as the sign-in's override window, so it was
+painted over most of the sign-in and became gamescope's keyboard focus window (Burnout Paradise
+Remastered, `gamescopectl focus_info`). On Windows a topmost window stays above every window that is neither topmost nor
+owned by it, so a window from another process that is neither is no longer an override candidate for
+a topmost focus window. Both windows' pids must be known; gamescope keeps -1 when XRes cannot name the
+client, which says nothing about the process. Same-process popups, topmost popups (WebView2 and CEF menus) and windows
+transient for the focus window are unchanged. gamescope read `_WINE_HWND_EXSTYLE` only from a
+PropertyNotify, and it listens for those from map on, so the style Wine sets before mapping a window
+was never known; `map_win` now reads it with the other properties, which the layered-launcher and
+Xalia rules rely on too. `tools/tests/test_gamescope_override_stacking.py` compiles the rule from
+this patch.

tools/gamescope/patches/0126-topmost-focus-outranks-other-process-override.patch

@@ -0,0 +1,43 @@
+diff --git a/src/steamcompmgr.cpp b/src/steamcompmgr.cpp
+--- a/src/steamcompmgr.cpp
++++ b/src/steamcompmgr.cpp
+@@ -4220,6 +4220,21 @@ static bool is_same_app_override_decorat
+ 		( candidate->hwndStyleEx & uDecorationStyle ) == uDecorationStyle;
+ }
+ 
++// Windows keeps a topmost window above every window that is neither topmost nor owned by it,
++// so another process's splash cannot cover a topmost sign-in (the EA app's, over its Steam launcher).
++static bool is_stacked_below_focus( steamcompmgr_win_t *candidate, steamcompmgr_win_t *focus )
++{
++	if ( candidate->pid <= 0 || focus->pid <= 0 || candidate->pid == focus->pid ||
++		 !candidate->hasHwndStyleEx || !focus->hasHwndStyleEx )
++		return false;
++
++	if ( candidate->type == steamcompmgr_win_type_t::XWAYLAND && focus->type == steamcompmgr_win_type_t::XWAYLAND &&
++		 candidate->xwayland().transientFor == focus->xwayland().id )
++		return false;
++
++	return ( focus->hwndStyleEx & WS_EX_TOPMOST ) && !( candidate->hwndStyleEx & WS_EX_TOPMOST );
++}
++
+ static bool is_good_override_candidate( steamcompmgr_win_t *override, steamcompmgr_win_t* focus )
+ {
+ 	// Some Chrome/Edge dropdowns (ie. FH5 xbox login) will automatically close themselves if you
+@@ -4235,6 +4250,9 @@ static bool is_good_override_candidate(
+ 	if ( is_same_app_override_decoration( override, focus ) )
+ 		return false;
+ 
++	if ( is_stacked_below_focus( override, focus ) )
++		return false;
++
+ 	if ( override->pid != focus->pid && !windows_share_app( override, focus ) )
+ 		return false;
+ 
+@@ -5836,6 +5854,7 @@ map_win(xwayland_ctx_t* ctx, Window id,
+ 	w->isSteamStreamingClient = get_prop(ctx, w->xwayland().id, ctx->atoms.steamStreamingClientAtom, 0);
+ 	w->isSteamStreamingClientVideo = get_prop(ctx, w->xwayland().id, ctx->atoms.steamStreamingClientVideoAtom, 0);
+ 	w->steamAppID = get_prop(ctx, w->xwayland().id, ctx->atoms.gameAtom, 0);
++	w->hwndStyleEx = get_prop(ctx, w->xwayland().id, ctx->atoms.wineHwndStyleEx, 0, &w->hasHwndStyleEx);
+ 
+ 	if ( steamMode == true )
+ 	{

tools/tests/test_gamescope_override_stacking.py

@@ -0,0 +1,101 @@
+from pathlib import Path
+import shutil
+import subprocess
+import tempfile
+import unittest
+
+
+ROOT = Path(__file__).resolve().parents[2]
+PATCH = ROOT / "tools/gamescope/patches/0126-topmost-focus-outranks-other-process-override.patch"
+
+
+def patched_rule():
+    added = [line[1:] for line in PATCH.read_text().splitlines() if line.startswith("+") and not line.startswith("+++")]
+    text = "\n".join(added)
+    start = text.index("static bool is_stacked_below_focus")
+    return text[start:text.index("\n}\n", start) + 3]
+
+
+PREAMBLE = r'''
+#include <cassert>
+#include <cstdint>
+#include <cstdio>
+
+static constexpr uint32_t WS_EX_TOPMOST = 0x00000008u;
+static constexpr uint32_t WS_EX_TOOLWINDOW = 0x00000080u;
+static constexpr uint32_t WS_EX_LAYERED = 0x00080000u;
+
+enum class steamcompmgr_win_type_t { XWAYLAND, XDG };
+struct xwayland_t { unsigned long id = 0; unsigned long transientFor = 0; };
+struct steamcompmgr_win_t {
+    int pid = 0;
+    bool hasHwndStyleEx = false;
+    uint32_t hwndStyleEx = 0;
+    steamcompmgr_win_type_t type = steamcompmgr_win_type_t::XWAYLAND;
+    xwayland_t x;
+    xwayland_t &xwayland() { return x; }
+};
+
+static steamcompmgr_win_t win(int pid, unsigned long id, uint32_t exstyle) {
+    steamcompmgr_win_t w;
+    w.pid = pid;
+    w.x.id = id;
+    w.hasHwndStyleEx = true;
+    w.hwndStyleEx = exstyle;
+    return w;
+}
+'''
+
+MAIN = r'''
+int main() {
+    steamcompmgr_win_t signin = win(4407, 0x2c00008, WS_EX_TOPMOST);
+    steamcompmgr_win_t splash = win(4055, 0x1c00005, WS_EX_LAYERED | WS_EX_TOOLWINDOW);
+    assert(is_stacked_below_focus(&splash, &signin));
+
+    steamcompmgr_win_t dropdown = win(4407, 0x2c00010, WS_EX_TOOLWINDOW);
+    assert(!is_stacked_below_focus(&dropdown, &signin));
+
+    steamcompmgr_win_t menu = win(5120, 0x3000004, WS_EX_TOPMOST | WS_EX_TOOLWINDOW);
+    assert(!is_stacked_below_focus(&menu, &signin));
+
+    steamcompmgr_win_t game = win(4407, 0x2c00008, 0);
+    assert(!is_stacked_below_focus(&splash, &game));
+
+    steamcompmgr_win_t owned = win(4055, 0x1c00007, 0);
+    owned.x.transientFor = signin.x.id;
+    assert(!is_stacked_below_focus(&owned, &signin));
+
+    steamcompmgr_win_t unknown = win(4055, 0x1c00009, 0);
+    unknown.hasHwndStyleEx = false;
+    assert(!is_stacked_below_focus(&unknown, &signin));
+
+    steamcompmgr_win_t no_pid = win(-1, 0x1c0000b, WS_EX_LAYERED | WS_EX_TOOLWINDOW);
+    assert(!is_stacked_below_focus(&no_pid, &signin));
+    steamcompmgr_win_t signin_no_pid = win(-1, 0x2c00008, WS_EX_TOPMOST);
+    assert(!is_stacked_below_focus(&splash, &signin_no_pid));
+
+    puts("ok");
+    return 0;
+}
+'''
+
+
+@unittest.skipUnless(shutil.which("c++"), "needs a C++ compiler")
+class OverrideStackingTest(unittest.TestCase):
+    def test_a_topmost_focus_keeps_other_process_windows_beneath_it(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            source = Path(tmp) / "stacking.cpp"
+            binary = Path(tmp) / "stacking"
+            source.write_text(PREAMBLE + patched_rule() + MAIN)
+            subprocess.run(["c++", "-std=c++17", "-Wall", "-Werror", str(source), "-o", str(binary)], check=True)
+            self.assertEqual(subprocess.run([str(binary)], capture_output=True, text=True, check=True).stdout.strip(), "ok")
+
+    def test_map_win_reads_the_wine_extended_style(self):
+        hunks = PATCH.read_text().split("\n@@ ")
+        map_win = next(hunk for hunk in hunks if "map_win(" in hunk.split("\n", 1)[0])
+        added = [line[1:].strip() for line in map_win.splitlines() if line.startswith("+")]
+        self.assertIn("w->hwndStyleEx = get_prop(ctx, w->xwayland().id, ctx->atoms.wineHwndStyleEx, 0, &w->hasHwndStyleEx);", added)
+
+
+if __name__ == "__main__":
+    unittest.main()