[build-tools] Record both iPhone Duo panels upright in record-sim

record-sim recorded only the largest framebuffer, which on iPhone Duo is the
inner panel, so recordings of a folded device were black. Record all
integrated panels side by side and rotate each one upright by the
nativeRotation in the device profile.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>

CHANGELOG.md

@@ -14,6 +14,7 @@ This is the log of notable changes to EAS CLI and related packages.
 ### 🐛 Bug fixes
 
 - [build-tools] Disable agent-device's internal idle timeouts in remote sessions, so only the session's own idle and duration limits end it. ([#4485](https://github.com/expo/eas-cli/pull/4485) by [@szdziedzic](https://github.com/szdziedzic))
+- [build-tools] Record both iPhone Duo panels side by side and upright, so iOS Simulator screen recordings are no longer black while the device is folded.
 
 ### 🧹 Chores
 

packages/build-tools/resources/record-sim/Package.swift

@@ -15,6 +15,7 @@ let package = Package(
         .target(
             name: "RecordSim",
             linkerSettings: [
+                .linkedFramework("Accelerate"),
                 .linkedFramework("AVFoundation"),
                 .linkedFramework("CoreGraphics"),
                 .linkedFramework("CoreMedia"),

packages/build-tools/resources/record-sim/README.md

@@ -9,6 +9,13 @@ It implements one recording path:
 3. Feed one continuous `AVAssetWriter`.
 4. Emit Apple HLS/CMAF-style fragmented MP4 segments.
 
+Foldable simulators such as iPhone Duo keep a framebuffer for every integrated
+panel. The recorder puts all integrated panels side by side in one video, in
+screen ID order, so the video follows the device when it folds. The panel that
+is not in use is black. Each panel is rotated upright by the `nativeRotation`
+in the device profile, which turns the iPhone Duo inner panel 90 degrees
+clockwise. Other simulators record their largest framebuffer.
+
 The output is suitable for uploading while recording. `session.json` contains
 the ordered segment metadata needed to build a playlist later.
 

packages/build-tools/resources/record-sim/Sources/RecordSim/FramebufferDisplaySource.swift

@@ -8,6 +8,18 @@ final class FramebufferDisplaySource {
         let width: Int
         let height: Int
         let seed: UInt32
+        /// Clockwise rotation in degrees (0, 90, 180, or 270) that shows the panel upright.
+        let rotation: Int
+
+        var uprightWidth: Int { rotation % 180 == 0 ? width : height }
+        var uprightHeight: Int { rotation % 180 == 0 ? height : width }
+    }
+
+    /// The panels to record, in a stable order. A nil entry is a panel that has no surface yet.
+    struct FrameSnapshot {
+        let panels: [SurfaceSnapshot?]
+
+        var seeds: [UInt32?] { panels.map { $0?.seed } }
     }
 
     private let deviceUDID: String
@@ -15,6 +27,10 @@ final class FramebufferDisplaySource {
     private let onFrame: () -> Void
     private let onSurfaceChange: () -> Void
     private var descriptors: [NSObject] = []
+    /// Integrated panels in screen ID order, which puts the iPhone Duo front panel first.
+    private var integratedPanels: [(descriptor: NSObject, rotation: Int)] = []
+    /// Mounting rotation of each integrated panel from the device profile, by screen ID.
+    private var nativeRotations: [UInt32: Int] = [:]
     private var callbackUUIDs: [ObjectIdentifier: NSUUID] = [:]
     private var retainedBlocks: [AnyObject] = []
     private var ioClient: NSObject?
@@ -46,40 +62,54 @@ final class FramebufferDisplaySource {
             throw RecorderError.make(3, "Failed to get simulator IO client")
         }
         ioClient = io
+        nativeRotations = Self.nativeRotations(device: device)
         try wireUpFramebuffer()
     }
 
     func stop() {
         unregisterCallbacks()
         descriptors.removeAll()
+        integratedPanels.removeAll()
         retainedBlocks.removeAll()
         ioClient = nil
     }
 
-    func surfaceSnapshot() -> SurfaceSnapshot? {
-        let surfaceSelector = NSSelectorFromString("framebufferSurface")
-        var bestSnapshot: SurfaceSnapshot?
-        var bestArea = 0
-        for descriptor in descriptors {
-            guard let surfaceObject = descriptor.perform(surfaceSelector)?.takeUnretainedValue()
-            else {
-                continue
-            }
-            let surface = unsafeBitCast(surfaceObject, to: IOSurface.self)
-            let width = IOSurfaceGetWidth(surface)
-            let height = IOSurfaceGetHeight(surface)
-            let area = width * height
-            if area > bestArea {
-                bestSnapshot = SurfaceSnapshot(
-                    surface: surface,
-                    width: width,
-                    height: height,
-                    seed: IOSurfaceGetSeed(surface)
-                )
-                bestArea = area
+    /// Foldable simulators such as iPhone Duo keep an IOSurface alive for every integrated panel,
+    /// and the panel that is not in use stays black. Record all integrated panels so the video
+    /// follows a fold; other simulators record their largest surface.
+    func frameSnapshot() -> FrameSnapshot? {
+        if integratedPanels.count >= 2 {
+            let panels = integratedPanels.map {
+                surfaceSnapshot(for: $0.descriptor, rotation: $0.rotation)
             }
+            return panels.contains { $0 != nil } ? FrameSnapshot(panels: panels) : nil
         }
-        return bestSnapshot
+        let largest = descriptors.compactMap { surfaceSnapshot(for: $0, rotation: 0) }.max {
+            $0.width * $0.height < $1.width * $1.height
+        }
+        return largest.map { FrameSnapshot(panels: [$0]) }
+    }
+
+    private func surfaceSnapshot(for descriptor: NSObject, rotation: Int) -> SurfaceSnapshot? {
+        guard
+            let surfaceObject = descriptor.perform(NSSelectorFromString("framebufferSurface"))?
+                .takeUnretainedValue()
+        else {
+            return nil
+        }
+        let surface = unsafeBitCast(surfaceObject, to: IOSurface.self)
+        let width = IOSurfaceGetWidth(surface)
+        let height = IOSurfaceGetHeight(surface)
+        guard width > 0, height > 0 else {
+            return nil
+        }
+        return SurfaceSnapshot(
+            surface: surface,
+            width: width,
+            height: height,
+            seed: IOSurfaceGetSeed(surface),
+            rotation: rotation
+        )
     }
 
     func rewireFramebuffer() throws {
@@ -94,6 +124,12 @@ final class FramebufferDisplaySource {
         let nextDescriptors = try findFramebufferDescriptors(io: io)
         unregisterCallbacks()
         descriptors = nextDescriptors
+        integratedPanels = Self.integratedPanels(in: nextDescriptors).map { panel in
+            // A panel mounted at 270 degrees, such as the iPhone Duo inner panel, turns 90
+            // degrees clockwise to be upright.
+            let nativeRotation = nativeRotations[panel.screenID] ?? 0
+            return (panel.descriptor, (360 - nativeRotation) % 360)
+        }
         retainedBlocks.removeAll()
         do {
             for descriptor in descriptors {
@@ -102,6 +138,7 @@ final class FramebufferDisplaySource {
         } catch {
             unregisterCallbacks()
             descriptors.removeAll()
+            integratedPanels.removeAll()
             retainedBlocks.removeAll()
             throw error
         }
@@ -135,6 +172,61 @@ final class FramebufferDisplaySource {
         return candidates
     }
 
+    /// Xcode releases without the screen properties API report no integrated panels, which keeps
+    /// them on the largest-surface path.
+    private static func integratedPanels(
+        in descriptors: [NSObject]
+    ) -> [(screenID: UInt32, descriptor: NSObject)] {
+        typealias Panel = (screenID: UInt32, descriptor: NSObject)
+        let panels = descriptors.compactMap { descriptor -> Panel? in
+            let selector = NSSelectorFromString("screenProperties")
+            guard descriptor.responds(to: selector),
+                let properties = descriptor.perform(selector)?.takeUnretainedValue()
+            else {
+                return nil
+            }
+            let object: AnyObject = properties
+            guard object.recordSimScreenType?() == integratedScreenType,
+                let screenID = object.recordSimScreenID?()
+            else {
+                return nil
+            }
+            return (screenID, descriptor)
+        }
+        return panels.sorted { $0.screenID < $1.screenID }
+    }
+
+    /// The device profile that serve-sim also reads. A missing profile means no rotation.
+    private static func nativeRotations(device: NSObject) -> [UInt32: Int] {
+        let typeSelector = NSSelectorFromString("deviceType")
+        let capabilitiesSelector = NSSelectorFromString("capabilities")
+        guard device.responds(to: typeSelector),
+            let type = device.perform(typeSelector)?.takeUnretainedValue() as? NSObject,
+            type.responds(to: capabilitiesSelector),
+            let profile = type.perform(capabilitiesSelector)?.takeUnretainedValue()
+                as? [String: Any],
+            let capabilities = profile["capabilities"] as? [String: Any],
+            let displays = capabilities["displays"] as? [[String: Any]]
+        else {
+            return [:]
+        }
+        var rotations: [UInt32: Int] = [:]
+        for display in displays {
+            guard display["displayType"] as? String == "integrated",
+                let id = display["screenID"] as? NSNumber,
+                let screenID = UInt32(exactly: id.int64Value),
+                let rotation = display["nativeRotation"] as? NSNumber,
+                [0, 90, 180, 270].contains(rotation.intValue)
+            else {
+                continue
+            }
+            rotations[screenID] = rotation.intValue
+        }
+        return rotations
+    }
+
+    private static let integratedScreenType: UInt64 = 0
+
     private func registerCallbacks(descriptor: NSObject) throws {
         let selector = NSSelectorFromString(
             "registerScreenCallbacksWithUUID:callbackQueue:frameCallback:surfacesChangedCallback:propertiesChangedCallback:"
@@ -190,3 +282,10 @@ final class FramebufferDisplaySource {
         callbackUUIDs.removeAll()
     }
 }
+
+/// Getters from the CoreSimDeviceIO SimScreenProperties protocol. Optional dispatch keeps Xcode
+/// releases that lack them working.
+@objc private protocol SimScreenPropertiesAccess {
+    @objc(screenID) func recordSimScreenID() -> UInt32
+    @objc(screenType) func recordSimScreenType() -> UInt64
+}

packages/build-tools/resources/record-sim/Sources/RecordSim/SimulatorRecorder.swift

@@ -1,4 +1,5 @@
 import AVFoundation
+import Accelerate
 import CoreVideo
 import Foundation
 import IOSurface
@@ -27,10 +28,9 @@ public final class SimulatorRecorder {
     private var monotonicClock = MonotonicClock()
     private var firstAcceptedCaptureTime: CMTime?
     private var firstAcceptedWallClock: Date?
-    private var recordingWidth: Int?
-    private var recordingHeight: Int?
+    private var canvasLayout: PanelCanvasLayout?
     private var lastPTS: CMTime?
-    private var lastSeed: UInt32?
+    private var lastSeeds: [UInt32?]?
     private var lastFrameCallbackElapsed: TimeInterval?
     private var lastFirstFrameRewireElapsed: TimeInterval = 0
     private var lastAppendedPixelBuffer: CVPixelBuffer?
@@ -147,14 +147,14 @@ public final class SimulatorRecorder {
             guard let firstAcceptedWallClock else {
                 throw RecorderError.make(25, "Missing first frame wall-clock timestamp")
             }
-            guard let recordingWidth, let recordingHeight else {
+            guard let canvasLayout else {
                 throw RecorderError.make(27, "Missing recording dimensions")
             }
             return try outputWriter.writeManifest(
                 configuration: configuration,
                 firstFrameWallClock: firstAcceptedWallClock,
-                width: recordingWidth,
-                height: recordingHeight
+                width: canvasLayout.width,
+                height: canvasLayout.height
             )
         }
     }
@@ -239,19 +239,13 @@ public final class SimulatorRecorder {
         if reason == .callback {
             lastFrameCallbackElapsed = monotonicClock.elapsedSeconds()
         }
-        guard let snapshot = displaySource?.surfaceSnapshot() else {
-            return
-        }
-        let surface = snapshot.surface
-        let surfaceWidth = snapshot.width
-        let surfaceHeight = snapshot.height
-        guard surfaceWidth > 0, surfaceHeight > 0 else {
+        guard let snapshot = displaySource?.frameSnapshot() else {
             return
         }
 
-        let seed = snapshot.seed
-        let matchesLastAppendedSeed = writerQueue.sync { lastSeed == seed }
-        if !force, matchesLastAppendedSeed {
+        let seeds = snapshot.seeds
+        let matchesLastAppendedSeeds = writerQueue.sync { lastSeeds == seeds }
+        if !force, matchesLastAppendedSeeds {
             return
         }
 
@@ -262,14 +256,10 @@ public final class SimulatorRecorder {
         let capturedAt = monotonicClock.elapsedTime()
         let capturedAtWallClock = Date()
         do {
-            let pixelBuffer = try copySurfaceToOwnedPixelBuffer(
-                surface,
-                width: surfaceWidth,
-                height: surfaceHeight
-            )
+            let pixelBuffer = try copyFrameToOwnedPixelBuffer(snapshot)
             appendOwned(
                 pixelBuffer: pixelBuffer,
-                seed: seed,
+                seeds: seeds,
                 capturedAt: capturedAt,
                 capturedAtWallClock: capturedAtWallClock
             )
@@ -296,17 +286,14 @@ public final class SimulatorRecorder {
             return
         }
 
-        guard let snapshot = displaySource.surfaceSnapshot(),
-            snapshot.width > 0,
-            snapshot.height > 0
-        else {
+        guard let snapshot = displaySource.frameSnapshot() else {
             rewireFramebuffer()
             return
         }
 
-        let observedSeed = snapshot.seed
-        let lastAppendedSeed = writerQueue.sync { lastSeed }
-        guard let lastAppendedSeed, observedSeed != lastAppendedSeed else {
+        let observedSeeds = snapshot.seeds
+        let lastAppendedSeeds = writerQueue.sync { lastSeeds }
+        guard let lastAppendedSeeds, observedSeeds != lastAppendedSeeds else {
             return
         }
 
@@ -369,22 +356,20 @@ public final class SimulatorRecorder {
         pendingLock.unlock()
     }
 
-    private func copySurfaceToOwnedPixelBuffer(
-        _ surface: IOSurface,
-        width: Int,
-        height: Int
+    private func copyFrameToOwnedPixelBuffer(
+        _ snapshot: FramebufferDisplaySource.FrameSnapshot
     ) throws -> CVPixelBuffer {
-        let pool = try writerQueue.sync {
+        let (pool, layout) = try writerQueue.sync {
             if let firstError {
                 throw firstError
             }
             if writer == nil {
-                try startWriter(width: width, height: height)
+                try startWriter(layout: PanelCanvasLayout(panels: snapshot.panels))
             }
-            guard let pool = adaptor?.pixelBufferPool else {
+            guard let pool = adaptor?.pixelBufferPool, let canvasLayout else {
                 throw RecorderError.make(40, "AVAssetWriter pixel buffer pool is unavailable")
             }
-            return pool
+            return (pool, canvasLayout)
         }
 
         var output: CVPixelBuffer?
@@ -397,39 +382,129 @@ public final class SimulatorRecorder {
         guard destinationLockStatus == kCVReturnSuccess else {
             throw RecorderError.make(50, "Failed to lock pixel buffer: \(destinationLockStatus)")
         }
-        let surfaceLockStatus = IOSurfaceLock(surface, .readOnly, nil)
-        guard surfaceLockStatus == KERN_SUCCESS else {
-            CVPixelBufferUnlockBaseAddress(destination, [])
-            throw RecorderError.make(49, "Failed to lock IOSurface: \(surfaceLockStatus)")
-        }
-        defer {
-            IOSurfaceUnlock(surface, .readOnly, nil)
-            CVPixelBufferUnlockBaseAddress(destination, [])
-        }
-
+        defer { CVPixelBufferUnlockBaseAddress(destination, []) }
         guard let destinationAddress = CVPixelBufferGetBaseAddress(destination) else {
             throw RecorderError.make(42, "Pixel buffer has no base address")
         }
-        let sourceAddress = IOSurfaceGetBaseAddress(surface)
-        let sourceStride = IOSurfaceGetBytesPerRow(surface)
+
         let destinationStride = CVPixelBufferGetBytesPerRow(destination)
-        let copyWidth = min(width, IOSurfaceGetWidth(surface), CVPixelBufferGetWidth(destination))
-        let copyHeight = min(
-            height, IOSurfaceGetHeight(surface), CVPixelBufferGetHeight(destination))
-        let copyBytes = min(copyWidth * 4, sourceStride, destinationStride)
-        for row in 0..<copyHeight {
-            memcpy(
-                destinationAddress.advanced(by: row * destinationStride),
-                sourceAddress.advanced(by: row * sourceStride),
-                copyBytes
+        let destinationWidth = CVPixelBufferGetWidth(destination)
+        let destinationHeight = CVPixelBufferGetHeight(destination)
+        for (index, slot) in layout.slots.enumerated() {
+            let slotWidth = max(0, min(slot.width, destinationWidth - slot.x))
+            guard slotWidth > 0 else {
+                continue
+            }
+            let panel = index < snapshot.panels.count ? snapshot.panels[index] : nil
+            try copyPanel(
+                panel?.surface,
+                rotation: panel?.rotation ?? 0,
+                into: destinationAddress.advanced(by: slot.x * 4),
+                stride: destinationStride,
+                width: slotWidth,
+                height: destinationHeight
             )
         }
         return destination
     }
 
+    /// Pool buffers are reused, so every pixel of the region that the surface does not cover,
+    /// including the whole region of a panel without a surface, is cleared to black.
+    private func copyPanel(
+        _ surface: IOSurface?,
+        rotation: Int,
+        into destinationAddress: UnsafeMutableRawPointer,
+        stride destinationStride: Int,
+        width: Int,
+        height: Int
+    ) throws {
+        let regionBytes = width * 4
+        guard let surface else {
+            for row in 0..<height {
+                memset(destinationAddress.advanced(by: row * destinationStride), 0, regionBytes)
+            }
+            return
+        }
+
+        let surfaceLockStatus = IOSurfaceLock(surface, .readOnly, nil)
+        guard surfaceLockStatus == KERN_SUCCESS else {
+            throw RecorderError.make(49, "Failed to lock IOSurface: \(surfaceLockStatus)")
+        }
+        defer { IOSurfaceUnlock(surface, .readOnly, nil) }
+
+        let sourceAddress = IOSurfaceGetBaseAddress(surface)
+        let sourceStride = IOSurfaceGetBytesPerRow(surface)
+        if rotation != 0 {
+            try rotatePanel(
+                surface,
+                rotation: rotation,
+                into: destinationAddress,
+                stride: destinationStride,
+                width: width,
+                height: height
+            )
+            return
+        }
+        let copyHeight = min(height, IOSurfaceGetHeight(surface))
+        let copyBytes = min(min(width, IOSurfaceGetWidth(surface)) * 4, sourceStride)
+        for row in 0..<height {
+            let destinationRow = destinationAddress.advanced(by: row * destinationStride)
+            let copiedBytes = row < copyHeight ? copyBytes : 0
+            if copiedBytes > 0 {
+                memcpy(destinationRow, sourceAddress.advanced(by: row * sourceStride), copiedBytes)
+            }
+            if copiedBytes < regionBytes {
+                memset(destinationRow.advanced(by: copiedBytes), 0, regionBytes - copiedBytes)
+            }
+        }
+    }
+
+    /// Expects the surface to be locked. vImage centers a surface whose upright size differs from
+    /// the region and fills the rest with black.
+    private func rotatePanel(
+        _ surface: IOSurface,
+        rotation: Int,
+        into destinationAddress: UnsafeMutableRawPointer,
+        stride destinationStride: Int,
+        width: Int,
+        height: Int
+    ) throws {
+        let rotationConstant: Int
+        switch rotation {
+        case 90: rotationConstant = kRotate90DegreesClockwise
+        case 180: rotationConstant = kRotate180DegreesClockwise
+        case 270: rotationConstant = kRotate270DegreesClockwise
+        default: throw RecorderError.make(51, "Unsupported panel rotation: \(rotation)")
+        }
+        let surfaceWidth = IOSurfaceGetWidth(surface)
+        let surfaceHeight = IOSurfaceGetHeight(surface)
+        let rotatedHeight = min(height, rotation == 180 ? surfaceHeight : surfaceWidth)
+        var source = vImage_Buffer(
+            data: IOSurfaceGetBaseAddress(surface),
+            height: vImagePixelCount(surfaceHeight),
+            width: vImagePixelCount(surfaceWidth),
+            rowBytes: IOSurfaceGetBytesPerRow(surface)
+        )
+        var destination = vImage_Buffer(
+            data: destinationAddress,
+            height: vImagePixelCount(rotatedHeight),
+            width: vImagePixelCount(width),
+            rowBytes: destinationStride
+        )
+        var black: [UInt8] = [0, 0, 0, 0]
+        let error = vImageRotate90_ARGB8888(
+            &source, &destination, UInt8(rotationConstant), &black, vImage_Flags(kvImageNoFlags))
+        guard error == kvImageNoError else {
+            throw RecorderError.make(52, "Failed to rotate panel: \(error)")
+        }
+        for row in rotatedHeight..<height {
+            memset(destinationAddress.advanced(by: row * destinationStride), 0, width * 4)
+        }
+    }
+
     private func appendOwned(
         pixelBuffer: CVPixelBuffer,
-        seed: UInt32,
+        seeds: [UInt32?],
         capturedAt: CMTime,
         capturedAtWallClock: Date
     ) {
@@ -441,7 +516,7 @@ public final class SimulatorRecorder {
             do {
                 try self.appendOnWriterQueue(
                     pixelBuffer: pixelBuffer,
-                    seed: seed,
+                    seeds: seeds,
                     capturedAt: capturedAt,
                     capturedAtWallClock: capturedAtWallClock
                 )
@@ -454,7 +529,7 @@ public final class SimulatorRecorder {
 
     private func appendOnWriterQueue(
         pixelBuffer: CVPixelBuffer,
-        seed: UInt32,
+        seeds: [UInt32?],
         capturedAt: CMTime,
         capturedAtWallClock: Date
     ) throws {
@@ -480,7 +555,7 @@ public final class SimulatorRecorder {
             firstAcceptedWallClock = capturedAtWallClock
         }
         lastPTS = pts
-        lastSeed = seed
+        lastSeeds = seeds
         lastAppendedPixelBuffer = pixelBuffer
         signalFirstFrameReadyIfNeeded()
     }
@@ -552,7 +627,9 @@ public final class SimulatorRecorder {
         return CMTimeSubtract(capturedAt, start)
     }
 
-    private func startWriter(width: Int, height: Int) throws {
+    private func startWriter(layout: PanelCanvasLayout) throws {
+        let width = layout.width
+        let height = layout.height
         guard let outputWriter else {
             throw RecorderError.make(46, "Missing output writer")
         }
@@ -600,8 +677,7 @@ public final class SimulatorRecorder {
         self.writer = writer
         self.input = input
         self.adaptor = adaptor
-        recordingWidth = width
-        recordingHeight = height
+        canvasLayout = layout
     }
 
     private func assetWriterSettings(width: Int, height: Int) -> [String: Any] {
@@ -642,3 +718,29 @@ public final class SimulatorRecorder {
         }
     }
 }
+
+/// Upright panels side by side, top aligned, sized from the first frame. A panel without a
+/// surface in that frame gets no region.
+private struct PanelCanvasLayout {
+    struct Slot {
+        let x: Int
+        let width: Int
+    }
+
+    let slots: [Slot]
+    let width: Int
+    let height: Int
+
+    init(panels: [FramebufferDisplaySource.SurfaceSnapshot?]) {
+        var x = 0
+        var slots: [Slot] = []
+        for panel in panels {
+            let width = panel?.uprightWidth ?? 0
+            slots.append(Slot(x: x, width: width))
+            x += width
+        }
+        self.slots = slots
+        width = x
+        height = panels.compactMap { $0?.uprightHeight }.max() ?? 0
+    }
+}