fix(drm): size a plane-rotated scanout as the mode transposed (#16444)

libs/scrap/src/common/drm_reader.rs

@@ -553,6 +553,12 @@ impl DrmReader {
         plane_rotation_answer(rc, rotation)
     }
 
+    /// The primary plane's rotation read NOW, for a caller that has not grabbed (enumeration):
+    /// `plane_rotation()` only reflects the last grab.
+    pub fn query_plane_rotation(&mut self) -> Option<u32> {
+        self.read_plane_rotation()
+    }
+
     /// Zero-copy EXPORT grab: fills a `drmtap_dmabuf_desc` (dma-buf fd, plane layout, HDR metadata) WITHOUT mapping, detiling or copying pixels, so on this
     /// path the root process never loads libEGL/libGLESv2. The exported fd is READ-ONLY (libdrmtap drops `DRM_RDWR` and `dup` shares that open file
     /// description), so the `--server` that receives it can map the scanout but never write the live framebuffer. Validation here is METADATA ONLY.

src/ipc/drm.rs

@@ -23,6 +23,10 @@ pub struct DrmDisplayInfo {
     /// alone is ambiguous across cards. Empty = the single auto-detected device.
     #[serde(default)]
     pub device: String,
+    /// The primary plane rotates its framebuffer by 90/270 in hardware, so every frame is this
+    /// mode TRANSPOSED whatever the compositor reports (gamescope's wl_output has no transform).
+    #[serde(default)]
+    pub plane_transposed: bool,
 }
 
 /// Mirrors `scrap::drm_reader::drmtap_dmabuf_desc` except `dma_buf_fd` (never serializes — it rides
@@ -280,6 +284,7 @@ fn drm_displays_from_reader(
             true
         })
         .map(|d| DrmDisplayInfo {
+            plane_transposed: plane_transposed(device, d.crtc_id),
             name: d.name,
             crtc_id: d.crtc_id,
             x: d.x,
@@ -294,6 +299,33 @@ fn drm_displays_from_reader(
     (displays, undriven)
 }
 
+const DRM_MODE_ROTATE_90: u32 = 0x2;
+const DRM_MODE_ROTATE_270: u32 = 0x8;
+
+/// Through a reader bound to `crtc_id`: the enumerating reader is bound to whichever crtc
+/// auto-select picked.
+fn plane_transposed(device: &str, crtc_id: u32) -> bool {
+    let dev = (!device.is_empty()).then_some(device);
+    let Some(mut r) = scrap::drm_reader::DrmReader::open(dev, crtc_id) else {
+        return false;
+    };
+    let transposed = match r.query_plane_rotation() {
+        Some(m) => m & (DRM_MODE_ROTATE_90 | DRM_MODE_ROTATE_270) != 0,
+        None => {
+            hbb_common::throttled_log!(
+                std::time::Duration::from_secs(600),
+                debug,
+                "drm: crtc {crtc_id} ({device:?}): plane rotation unreadable; sized as unrotated"
+            );
+            false
+        }
+    };
+    if transposed {
+        log::trace!("drm: crtc {crtc_id} ({device:?}) is rotated 90/270 by its plane");
+    }
+    transposed
+}
+
 /// Active displays of every DRM device + the connected-but-undriven identities, from ONE look.
 fn drm_enumerate_all_displays() -> (Vec<DrmDisplayInfo>, Vec<String>) {
     if let Some(devices) = scrap::drm_reader::list_devices() {
@@ -1673,6 +1705,7 @@ mod drm_conn_tests {
             active: true,
             render_node: "/dev/dri/renderD129".to_owned(),
             device: "/dev/dri/card2".to_owned(),
+            plane_transposed: false,
         };
         let wire = serde_json::to_vec(&current).unwrap();
         let back: DrmDisplayInfo = serde_json::from_slice(&wire).unwrap();

src/server/drm_capturer.rs

@@ -110,6 +110,17 @@ fn connector_key(d: &DrmDisplayInfo) -> String {
     format!("{}:{}", d.device, d.name)
 }
 
+/// The transform whose dimensions the delivered frames have: a plane rotated 90/270 in hardware
+/// already hands over the transposed scanout (and `frame_transform` then turns nothing), whatever
+/// the output transform says; otherwise the output transform is undone in software.
+fn delivered_transform(d: &DrmDisplayInfo, wl_transform: i32) -> i32 {
+    if d.plane_transposed {
+        90
+    } else {
+        wl_transform
+    }
+}
+
 /// Frame dimensions after undoing `transform` degrees of output rotation.
 fn rotated_dims(transform: i32, w: usize, h: usize) -> (usize, usize) {
     if transform == 90 || transform == 270 {
@@ -416,9 +427,13 @@ impl IpcDrmCapturer {
                 stop,
                 display,
                 connector: displays.get(wire_idx).map(connector_key),
-                session_size: displays
-                    .get(wire_idx)
-                    .map(|d| rotated_dims(wl_transform, d.width as usize, d.height as usize)),
+                session_size: displays.get(wire_idx).map(|d| {
+                    rotated_dims(
+                        delivered_transform(d, wl_transform),
+                        d.width as usize,
+                        d.height as usize,
+                    )
+                }),
                 transform: wl_transform,
                 snapshot_gen,
                 cur: Vec::new(),
@@ -2122,7 +2137,8 @@ fn augment_with_wayland_geometry_from(
         // Identity matches ONLY, the same rule the capturer's transform follows: swapping on a
         // layout-order guess advertises dimensions the capturer will not deliver.
         let is_identity = identity[i].is_some() && identity[i] == matched[i];
-        if is_identity && (w.transform == 90 || w.transform == 270) {
+        // A plane-transposed display was already swapped by `display_info_from_drm`.
+        if is_identity && !drm[i].plane_transposed && (w.transform == 90 || w.transform == 270) {
             std::mem::swap(&mut info.width, &mut info.height);
             info.original_resolution = super::display_service::get_original_resolution(
                 &drm[i].name,
@@ -2259,13 +2275,18 @@ fn swap_available_displays(list: Vec<DrmDisplayInfo>) {
 }
 
 fn display_info_from_drm(d: &DrmDisplayInfo) -> DisplayInfo {
+    let (width, height) = rotated_dims(
+        delivered_transform(d, 0),
+        d.width as usize,
+        d.height as usize,
+    );
     let original_resolution =
-        super::display_service::get_original_resolution(&d.name, d.width as usize, d.height as usize);
+        super::display_service::get_original_resolution(&d.name, width, height);
     DisplayInfo {
         x: d.x,
         y: d.y,
-        width: d.width as i32,
-        height: d.height as i32,
+        width: width as i32,
+        height: height as i32,
         name: d.name.clone(),
         online: d.active,
         cursor_embedded: false,
@@ -2330,7 +2351,11 @@ pub(super) fn get_capturer_info(
     // Origin and transform come from the ONE snapshot new() resolved, so both reflect the
     // same output assignment; dimensions stay PHYSICAL, rotated to frame orientation.
     let origin = origin.unwrap_or((d.x, d.y));
-    let (cap_w, cap_h) = rotated_dims(capturer.transform, d.width as usize, d.height as usize);
+    let (cap_w, cap_h) = rotated_dims(
+        delivered_transform(&d, capturer.transform),
+        d.width as usize,
+        d.height as usize,
+    );
     Ok(super::video_service::CapturerInfo {
         origin,
         width: cap_w,
@@ -3792,6 +3817,7 @@ mod drm_capturer_tests {
             active: true,
             render_node: String::new(),
             device: String::new(),
+            plane_transposed: false,
         }
     }
 
@@ -3814,6 +3840,51 @@ mod drm_capturer_tests {
         }
     }
 
+    fn plane_transposed_display(name: &str, w: u32, h: u32) -> DrmDisplayInfo {
+        DrmDisplayInfo {
+            plane_transposed: true,
+            ..drm_display(name, w, h)
+        }
+    }
+
+    #[test]
+    fn a_plane_transposed_display_advertises_its_scanout() {
+        // Steam Deck under gamescope: 800x1280 panel mode, 1280x800 framebuffer on a rotate-270
+        // plane, and no wayland output that matches the connector.
+        let drm = [plane_transposed_display("eDP-1", 800, 1280)];
+        let wl = scrap::wayland::display::Displays {
+            primary: 0,
+            displays: vec![],
+        };
+        let assignment = assign_wayland_outputs(&drm, &wl.displays);
+        let infos = augment_with_wayland_geometry_from(&drm, &wl, &assignment);
+        assert_eq!((infos[0].width, infos[0].height), (1280, 800));
+    }
+
+    #[test]
+    fn a_plane_transposed_scanout_is_delivered_at_the_advertised_size() {
+        let d = plane_transposed_display("eDP-1", 800, 1280);
+        let session = rotated_dims(delivered_transform(&d, 0), d.width as usize, d.height as usize);
+        let mut c = capturer_named(Some(session), Some("test:plane-transposed-session"));
+        put_frame_rot(&c, 1280, 800, Some(0x8));
+        assert!(matches!(c.frame(Duration::from_millis(50)), Ok(_)));
+    }
+
+    #[test]
+    fn a_rotated_output_over_a_rotated_plane_is_swapped_once() {
+        // mutter rotates the output and programs the plane from it: one quarter turn, as before.
+        let drm = [plane_transposed_display("HDMI-A-1", 1920, 1080)];
+        let mut out = wl_display("HDMI-1", 0, 0, 1920, 1080);
+        out.transform = 90;
+        let wl = scrap::wayland::display::Displays {
+            primary: 0,
+            displays: vec![out],
+        };
+        let assignment = assign_wayland_outputs(&drm, &wl.displays);
+        let infos = augment_with_wayland_geometry_from(&drm, &wl, &assignment);
+        assert_eq!((infos[0].width, infos[0].height), (1080, 1920));
+    }
+
     #[test]
     fn a_lone_rotated_output_advertises_delivered_dimensions() {
         // Fix for the origin-only cut: one connector, one rotated output. The capturer will